2024-11-18 07:51:56,787 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-18 07:51:56,801 main DEBUG Took 0.012440 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 07:51:56,802 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 07:51:56,802 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 07:51:56,804 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 07:51:56,805 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 07:51:56,815 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 07:51:56,827 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 07:51:56,828 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 07:51:56,828 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 07:51:56,829 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 07:51:56,829 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 07:51:56,830 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 07:51:56,831 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 07:51:56,831 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 07:51:56,832 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 07:51:56,832 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 07:51:56,834 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 07:51:56,834 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 07:51:56,834 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 07:51:56,835 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 07:51:56,835 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 07:51:56,836 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 07:51:56,836 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 07:51:56,836 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 07:51:56,837 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 07:51:56,837 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 07:51:56,837 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 07:51:56,838 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 07:51:56,838 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 07:51:56,839 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 07:51:56,839 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 07:51:56,839 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 07:51:56,841 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 07:51:56,843 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 07:51:56,846 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 07:51:56,847 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 07:51:56,848 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 07:51:56,849 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 07:51:56,859 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 07:51:56,862 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 07:51:56,865 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 07:51:56,865 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 07:51:56,866 main DEBUG createAppenders(={Console}) 2024-11-18 07:51:56,867 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-18 07:51:56,867 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-18 07:51:56,868 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-18 07:51:56,868 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 07:51:56,869 main DEBUG OutputStream closed 2024-11-18 07:51:56,869 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 07:51:56,870 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 07:51:56,870 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-18 07:51:56,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 07:51:56,965 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 07:51:56,966 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 07:51:56,968 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 07:51:56,969 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 07:51:56,969 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 07:51:56,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 07:51:56,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 07:51:56,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 07:51:56,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 07:51:56,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 07:51:56,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 07:51:56,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 07:51:56,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 07:51:56,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 07:51:56,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 07:51:56,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 07:51:56,976 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 07:51:56,979 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 07:51:56,980 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-18 07:51:56,980 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 07:51:56,981 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-18T07:51:57,327 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7 2024-11-18 07:51:57,331 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 07:51:57,332 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T07:51:57,343 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-18T07:51:57,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-18T07:51:57,393 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/cluster_477ae96c-b03e-42a8-3641-251bd626575f, deleteOnExit=true 2024-11-18T07:51:57,393 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-18T07:51:57,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/test.cache.data in system properties and HBase conf 2024-11-18T07:51:57,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/hadoop.tmp.dir in system properties and HBase conf 2024-11-18T07:51:57,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/hadoop.log.dir in system properties and HBase conf 2024-11-18T07:51:57,398 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-18T07:51:57,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-18T07:51:57,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-18T07:51:57,516 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T07:51:57,641 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T07:51:57,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-18T07:51:57,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-18T07:51:57,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-18T07:51:57,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T07:51:57,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-18T07:51:57,652 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-18T07:51:57,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T07:51:57,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T07:51:57,654 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-18T07:51:57,655 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/nfs.dump.dir in system properties and HBase conf 2024-11-18T07:51:57,656 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/java.io.tmpdir in system properties and HBase conf 2024-11-18T07:51:57,656 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T07:51:57,657 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-18T07:51:57,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-18T07:51:58,756 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-18T07:51:58,835 INFO [Time-limited test {}] log.Log(170): Logging initialized @2892ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-18T07:51:58,926 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T07:51:59,003 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T07:51:59,028 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T07:51:59,028 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T07:51:59,030 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T07:51:59,049 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T07:51:59,053 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/hadoop.log.dir/,AVAILABLE} 2024-11-18T07:51:59,055 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T07:51:59,277 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/java.io.tmpdir/jetty-localhost-35001-hadoop-hdfs-3_4_1-tests_jar-_-any-5835621813375237192/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T07:51:59,285 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:35001} 2024-11-18T07:51:59,285 INFO [Time-limited test {}] server.Server(415): Started @3342ms 2024-11-18T07:52:00,025 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T07:52:00,033 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T07:52:00,034 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T07:52:00,034 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T07:52:00,034 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T07:52:00,035 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72782cb0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/hadoop.log.dir/,AVAILABLE} 2024-11-18T07:52:00,036 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@186a2d75{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T07:52:00,183 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@651ab874{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/java.io.tmpdir/jetty-localhost-35467-hadoop-hdfs-3_4_1-tests_jar-_-any-981234785395691955/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T07:52:00,184 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6bf72caa{HTTP/1.1, (http/1.1)}{localhost:35467} 2024-11-18T07:52:00,184 INFO [Time-limited test {}] server.Server(415): Started @4241ms 2024-11-18T07:52:00,233 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T07:52:01,139 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/cluster_477ae96c-b03e-42a8-3641-251bd626575f/data/data1/current/BP-1093338534-172.17.0.2-1731916318289/current, will proceed with Du for space computation calculation, 2024-11-18T07:52:01,139 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/cluster_477ae96c-b03e-42a8-3641-251bd626575f/data/data2/current/BP-1093338534-172.17.0.2-1731916318289/current, will proceed with Du for space computation calculation, 2024-11-18T07:52:01,177 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T07:52:01,240 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7 2024-11-18T07:52:01,242 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf1f47a0b484fa358 with lease ID 0x6a812bdbedff21b2: Processing first storage report for DS-54035976-ce14-4282-869f-c28be41b513f from datanode DatanodeRegistration(127.0.0.1:40631, datanodeUuid=392a6dfc-8655-4f40-a45a-57258804d6c2, infoPort=39457, infoSecurePort=0, ipcPort=40305, storageInfo=lv=-57;cid=testClusterID;nsid=1972313336;c=1731916318289) 2024-11-18T07:52:01,243 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf1f47a0b484fa358 with lease ID 0x6a812bdbedff21b2: from storage DS-54035976-ce14-4282-869f-c28be41b513f node DatanodeRegistration(127.0.0.1:40631, datanodeUuid=392a6dfc-8655-4f40-a45a-57258804d6c2, infoPort=39457, infoSecurePort=0, ipcPort=40305, storageInfo=lv=-57;cid=testClusterID;nsid=1972313336;c=1731916318289), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-18T07:52:01,244 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf1f47a0b484fa358 with lease ID 0x6a812bdbedff21b2: Processing first storage report for DS-ced6d565-8394-4d83-a309-fdca37199c2f from datanode DatanodeRegistration(127.0.0.1:40631, datanodeUuid=392a6dfc-8655-4f40-a45a-57258804d6c2, infoPort=39457, infoSecurePort=0, ipcPort=40305, storageInfo=lv=-57;cid=testClusterID;nsid=1972313336;c=1731916318289) 2024-11-18T07:52:01,244 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf1f47a0b484fa358 with lease ID 0x6a812bdbedff21b2: from storage DS-ced6d565-8394-4d83-a309-fdca37199c2f node DatanodeRegistration(127.0.0.1:40631, datanodeUuid=392a6dfc-8655-4f40-a45a-57258804d6c2, infoPort=39457, infoSecurePort=0, ipcPort=40305, storageInfo=lv=-57;cid=testClusterID;nsid=1972313336;c=1731916318289), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T07:52:01,360 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/cluster_477ae96c-b03e-42a8-3641-251bd626575f/zookeeper_0, clientPort=61891, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/cluster_477ae96c-b03e-42a8-3641-251bd626575f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/cluster_477ae96c-b03e-42a8-3641-251bd626575f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T07:52:01,370 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61891 2024-11-18T07:52:01,386 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T07:52:01,389 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T07:52:01,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741825_1001 (size=7) 2024-11-18T07:52:02,056 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3 with version=8 2024-11-18T07:52:02,057 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/hbase-staging 2024-11-18T07:52:02,174 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T07:52:02,462 INFO [Time-limited test {}] client.ConnectionUtils(128): master/87fb3d252950:0 server-side Connection retries=45 2024-11-18T07:52:02,471 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T07:52:02,472 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T07:52:02,478 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T07:52:02,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T07:52:02,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T07:52:02,663 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T07:52:02,728 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T07:52:02,737 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T07:52:02,740 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T07:52:02,765 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 87451 (auto-detected) 2024-11-18T07:52:02,766 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T07:52:02,783 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42633 2024-11-18T07:52:02,805 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42633 connecting to ZooKeeper ensemble=127.0.0.1:61891 2024-11-18T07:52:02,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:426330x0, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T07:52:02,919 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42633-0x1014e3765000000 connected 2024-11-18T07:52:02,993 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T07:52:02,996 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T07:52:03,007 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T07:52:03,012 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3, hbase.cluster.distributed=false 2024-11-18T07:52:03,041 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T07:52:03,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42633 2024-11-18T07:52:03,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42633 2024-11-18T07:52:03,049 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42633 2024-11-18T07:52:03,049 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42633 2024-11-18T07:52:03,050 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42633 2024-11-18T07:52:03,166 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/87fb3d252950:0 server-side Connection retries=45 2024-11-18T07:52:03,168 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T07:52:03,168 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T07:52:03,168 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T07:52:03,168 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T07:52:03,169 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T07:52:03,171 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T07:52:03,174 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T07:52:03,175 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37987 2024-11-18T07:52:03,177 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37987 connecting to ZooKeeper ensemble=127.0.0.1:61891 2024-11-18T07:52:03,178 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T07:52:03,181 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T07:52:03,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:379870x0, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T07:52:03,217 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:379870x0, quorum=127.0.0.1:61891, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T07:52:03,217 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37987-0x1014e3765000001 connected 2024-11-18T07:52:03,221 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T07:52:03,229 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T07:52:03,231 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T07:52:03,237 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T07:52:03,238 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37987 2024-11-18T07:52:03,238 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37987 2024-11-18T07:52:03,242 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37987 2024-11-18T07:52:03,246 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37987 2024-11-18T07:52:03,247 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37987 2024-11-18T07:52:03,261 DEBUG [M:0;87fb3d252950:42633 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;87fb3d252950:42633 2024-11-18T07:52:03,262 INFO [master/87fb3d252950:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/87fb3d252950,42633,1731916322278 2024-11-18T07:52:03,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T07:52:03,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T07:52:03,300 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/87fb3d252950,42633,1731916322278 2024-11-18T07:52:03,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T07:52:03,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T07:52:03,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T07:52:03,334 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T07:52:03,336 INFO [master/87fb3d252950:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/87fb3d252950,42633,1731916322278 from backup master directory 2024-11-18T07:52:03,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T07:52:03,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/87fb3d252950,42633,1731916322278 2024-11-18T07:52:03,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T07:52:03,348 WARN [master/87fb3d252950:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T07:52:03,349 INFO [master/87fb3d252950:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=87fb3d252950,42633,1731916322278 2024-11-18T07:52:03,351 INFO [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T07:52:03,353 INFO [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T07:52:03,423 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/hbase.id] with ID: 46703b93-0193-4c59-a818-58ceb8c8d03b 2024-11-18T07:52:03,423 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/.tmp/hbase.id 2024-11-18T07:52:03,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741826_1002 (size=42) 2024-11-18T07:52:03,836 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/.tmp/hbase.id]:[hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/hbase.id] 2024-11-18T07:52:03,891 INFO [master/87fb3d252950:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T07:52:03,898 INFO [master/87fb3d252950:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T07:52:03,924 INFO [master/87fb3d252950:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-18T07:52:03,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T07:52:03,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T07:52:03,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741827_1003 (size=196) 2024-11-18T07:52:04,016 INFO [master/87fb3d252950:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T07:52:04,020 INFO [master/87fb3d252950:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T07:52:04,043 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:04,048 INFO [master/87fb3d252950:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T07:52:04,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741828_1004 (size=1189) 2024-11-18T07:52:04,501 INFO [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store 2024-11-18T07:52:04,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741829_1005 (size=34) 2024-11-18T07:52:04,527 INFO [master/87fb3d252950:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T07:52:04,530 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:52:04,531 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T07:52:04,532 INFO [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T07:52:04,532 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T07:52:04,533 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T07:52:04,533 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T07:52:04,533 INFO [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T07:52:04,534 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731916324531Disabling compacts and flushes for region at 1731916324531Disabling writes for close at 1731916324533 (+2 ms)Writing region close event to WAL at 1731916324533Closed at 1731916324533 2024-11-18T07:52:04,537 WARN [master/87fb3d252950:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/.initializing 2024-11-18T07:52:04,537 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/WALs/87fb3d252950,42633,1731916322278 2024-11-18T07:52:04,546 INFO [master/87fb3d252950:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T07:52:04,563 INFO [master/87fb3d252950:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=87fb3d252950%2C42633%2C1731916322278, suffix=, logDir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/WALs/87fb3d252950,42633,1731916322278, archiveDir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/oldWALs, maxLogs=10 2024-11-18T07:52:04,589 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/WALs/87fb3d252950,42633,1731916322278/87fb3d252950%2C42633%2C1731916322278.1731916324568, exclude list is [], retry=0 2024-11-18T07:52:04,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40631,DS-54035976-ce14-4282-869f-c28be41b513f,DISK] 2024-11-18T07:52:04,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-18T07:52:04,659 INFO [master/87fb3d252950:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/WALs/87fb3d252950,42633,1731916322278/87fb3d252950%2C42633%2C1731916322278.1731916324568 2024-11-18T07:52:04,661 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39457:39457)] 2024-11-18T07:52:04,661 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T07:52:04,662 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:52:04,665 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T07:52:04,666 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T07:52:04,712 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T07:52:04,743 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T07:52:04,748 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:04,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T07:52:04,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T07:52:04,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T07:52:04,755 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:04,757 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:52:04,757 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T07:52:04,760 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T07:52:04,760 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:04,761 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:52:04,762 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T07:52:04,765 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T07:52:04,765 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:04,766 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:52:04,766 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T07:52:04,770 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T07:52:04,771 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T07:52:04,777 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T07:52:04,778 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T07:52:04,782 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T07:52:04,787 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T07:52:04,793 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T07:52:04,794 INFO [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73709308, jitterRate=0.0983542799949646}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T07:52:04,803 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731916324678Initializing all the Stores at 1731916324681 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731916324682 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916324682Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916324683 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916324683Cleaning up temporary data from old regions at 1731916324778 (+95 ms)Region opened successfully at 1731916324803 (+25 ms) 2024-11-18T07:52:04,805 INFO [master/87fb3d252950:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T07:52:04,841 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343705bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=87fb3d252950/172.17.0.2:0 2024-11-18T07:52:04,881 INFO [master/87fb3d252950:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T07:52:04,896 INFO [master/87fb3d252950:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T07:52:04,896 INFO [master/87fb3d252950:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T07:52:04,900 INFO [master/87fb3d252950:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T07:52:04,902 INFO [master/87fb3d252950:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-18T07:52:04,908 INFO [master/87fb3d252950:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-18T07:52:04,908 INFO [master/87fb3d252950:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T07:52:04,934 INFO [master/87fb3d252950:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T07:52:04,946 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T07:52:04,990 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T07:52:04,994 INFO [master/87fb3d252950:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T07:52:04,996 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T07:52:05,007 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T07:52:05,010 INFO [master/87fb3d252950:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T07:52:05,015 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T07:52:05,024 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T07:52:05,025 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T07:52:05,032 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T07:52:05,049 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T07:52:05,057 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T07:52:05,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T07:52:05,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T07:52:05,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T07:52:05,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T07:52:05,074 INFO [master/87fb3d252950:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=87fb3d252950,42633,1731916322278, sessionid=0x1014e3765000000, setting cluster-up flag (Was=false) 2024-11-18T07:52:05,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T07:52:05,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T07:52:05,132 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T07:52:05,135 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=87fb3d252950,42633,1731916322278 2024-11-18T07:52:05,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T07:52:05,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T07:52:05,182 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T07:52:05,184 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=87fb3d252950,42633,1731916322278 2024-11-18T07:52:05,192 INFO [master/87fb3d252950:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T07:52:05,253 INFO [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(746): ClusterId : 46703b93-0193-4c59-a818-58ceb8c8d03b 2024-11-18T07:52:05,256 DEBUG [RS:0;87fb3d252950:37987 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T07:52:05,274 DEBUG [RS:0;87fb3d252950:37987 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T07:52:05,274 DEBUG [RS:0;87fb3d252950:37987 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T07:52:05,283 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T07:52:05,284 DEBUG [RS:0;87fb3d252950:37987 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T07:52:05,284 DEBUG [RS:0;87fb3d252950:37987 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bda69ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=87fb3d252950/172.17.0.2:0 2024-11-18T07:52:05,296 INFO [master/87fb3d252950:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T07:52:05,303 DEBUG [RS:0;87fb3d252950:37987 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;87fb3d252950:37987 2024-11-18T07:52:05,305 INFO [master/87fb3d252950:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T07:52:05,308 INFO [RS:0;87fb3d252950:37987 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T07:52:05,308 INFO [RS:0;87fb3d252950:37987 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T07:52:05,308 DEBUG [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T07:52:05,311 INFO [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(2659): reportForDuty to master=87fb3d252950,42633,1731916322278 with port=37987, startcode=1731916323128 2024-11-18T07:52:05,312 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 87fb3d252950,42633,1731916322278 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T07:52:05,321 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/87fb3d252950:0, corePoolSize=5, maxPoolSize=5 2024-11-18T07:52:05,321 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/87fb3d252950:0, corePoolSize=5, maxPoolSize=5 2024-11-18T07:52:05,322 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/87fb3d252950:0, corePoolSize=5, maxPoolSize=5 2024-11-18T07:52:05,322 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/87fb3d252950:0, corePoolSize=5, maxPoolSize=5 2024-11-18T07:52:05,322 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/87fb3d252950:0, corePoolSize=10, maxPoolSize=10 2024-11-18T07:52:05,323 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/87fb3d252950:0, corePoolSize=1, maxPoolSize=1 2024-11-18T07:52:05,323 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/87fb3d252950:0, corePoolSize=2, maxPoolSize=2 2024-11-18T07:52:05,323 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/87fb3d252950:0, corePoolSize=1, maxPoolSize=1 2024-11-18T07:52:05,325 DEBUG [RS:0;87fb3d252950:37987 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T07:52:05,333 INFO [master/87fb3d252950:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731916355333 2024-11-18T07:52:05,334 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T07:52:05,334 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T07:52:05,335 INFO [master/87fb3d252950:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T07:52:05,336 INFO [master/87fb3d252950:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T07:52:05,341 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:05,341 INFO [master/87fb3d252950:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T07:52:05,341 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T07:52:05,341 INFO [master/87fb3d252950:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T07:52:05,342 INFO [master/87fb3d252950:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T07:52:05,342 INFO [master/87fb3d252950:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T07:52:05,347 INFO [master/87fb3d252950:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:05,354 INFO [master/87fb3d252950:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T07:52:05,355 INFO [master/87fb3d252950:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T07:52:05,356 INFO [master/87fb3d252950:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T07:52:05,367 INFO [master/87fb3d252950:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T07:52:05,367 INFO [master/87fb3d252950:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T07:52:05,370 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/87fb3d252950:0:becomeActiveMaster-HFileCleaner.large.0-1731916325368,5,FailOnTimeoutGroup] 2024-11-18T07:52:05,372 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/87fb3d252950:0:becomeActiveMaster-HFileCleaner.small.0-1731916325371,5,FailOnTimeoutGroup] 2024-11-18T07:52:05,372 INFO [master/87fb3d252950:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:05,372 INFO [master/87fb3d252950:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T07:52:05,375 INFO [master/87fb3d252950:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:05,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741831_1007 (size=1321) 2024-11-18T07:52:05,376 INFO [master/87fb3d252950:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:05,380 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T07:52:05,380 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43511, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T07:52:05,380 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3 2024-11-18T07:52:05,388 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42633 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 87fb3d252950,37987,1731916323128 2024-11-18T07:52:05,390 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42633 {}] master.ServerManager(517): Registering regionserver=87fb3d252950,37987,1731916323128 2024-11-18T07:52:05,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741832_1008 (size=32) 2024-11-18T07:52:05,396 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:52:05,399 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T07:52:05,402 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T07:52:05,403 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:05,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T07:52:05,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T07:52:05,407 DEBUG [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3 2024-11-18T07:52:05,408 DEBUG [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44371 2024-11-18T07:52:05,408 DEBUG [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T07:52:05,408 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T07:52:05,408 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:05,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T07:52:05,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T07:52:05,415 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T07:52:05,415 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:05,417 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T07:52:05,417 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T07:52:05,420 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T07:52:05,420 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:05,421 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T07:52:05,422 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T07:52:05,424 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740 2024-11-18T07:52:05,425 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740 2024-11-18T07:52:05,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T07:52:05,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T07:52:05,430 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T07:52:05,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T07:52:05,438 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T07:52:05,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T07:52:05,439 DEBUG [RS:0;87fb3d252950:37987 {}] zookeeper.ZKUtil(111): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/87fb3d252950,37987,1731916323128 2024-11-18T07:52:05,439 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74909982, jitterRate=0.11624571681022644}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T07:52:05,440 WARN [RS:0;87fb3d252950:37987 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T07:52:05,440 INFO [RS:0;87fb3d252950:37987 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T07:52:05,440 DEBUG [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/WALs/87fb3d252950,37987,1731916323128 2024-11-18T07:52:05,445 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731916325396Initializing all the Stores at 1731916325398 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731916325398Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731916325399 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916325399Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731916325399Cleaning up temporary data from old regions at 1731916325429 (+30 ms)Region opened successfully at 1731916325445 (+16 ms) 2024-11-18T07:52:05,446 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [87fb3d252950,37987,1731916323128] 2024-11-18T07:52:05,446 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T07:52:05,446 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T07:52:05,446 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T07:52:05,446 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T07:52:05,446 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T07:52:05,452 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T07:52:05,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731916325446Disabling compacts and flushes for region at 1731916325446Disabling writes for close at 1731916325446Writing region close event to WAL at 1731916325451 (+5 ms)Closed at 1731916325451 2024-11-18T07:52:05,456 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T07:52:05,456 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T07:52:05,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T07:52:05,471 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T07:52:05,474 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T07:52:05,475 INFO [RS:0;87fb3d252950:37987 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T07:52:05,492 INFO [RS:0;87fb3d252950:37987 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T07:52:05,497 INFO [RS:0;87fb3d252950:37987 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T07:52:05,498 INFO [RS:0;87fb3d252950:37987 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:05,499 INFO [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T07:52:05,508 INFO [RS:0;87fb3d252950:37987 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T07:52:05,510 INFO [RS:0;87fb3d252950:37987 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:05,510 DEBUG [RS:0;87fb3d252950:37987 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/87fb3d252950:0, corePoolSize=1, maxPoolSize=1 2024-11-18T07:52:05,511 DEBUG [RS:0;87fb3d252950:37987 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/87fb3d252950:0, corePoolSize=1, maxPoolSize=1 2024-11-18T07:52:05,511 DEBUG [RS:0;87fb3d252950:37987 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/87fb3d252950:0, corePoolSize=1, maxPoolSize=1 2024-11-18T07:52:05,511 DEBUG [RS:0;87fb3d252950:37987 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/87fb3d252950:0, corePoolSize=1, maxPoolSize=1 2024-11-18T07:52:05,511 DEBUG [RS:0;87fb3d252950:37987 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/87fb3d252950:0, corePoolSize=1, maxPoolSize=1 2024-11-18T07:52:05,511 DEBUG [RS:0;87fb3d252950:37987 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/87fb3d252950:0, corePoolSize=2, maxPoolSize=2 2024-11-18T07:52:05,512 DEBUG [RS:0;87fb3d252950:37987 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0, corePoolSize=1, maxPoolSize=1 2024-11-18T07:52:05,512 DEBUG [RS:0;87fb3d252950:37987 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/87fb3d252950:0, corePoolSize=1, maxPoolSize=1 2024-11-18T07:52:05,512 DEBUG [RS:0;87fb3d252950:37987 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/87fb3d252950:0, corePoolSize=1, maxPoolSize=1 2024-11-18T07:52:05,512 DEBUG [RS:0;87fb3d252950:37987 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/87fb3d252950:0, corePoolSize=1, maxPoolSize=1 2024-11-18T07:52:05,512 DEBUG [RS:0;87fb3d252950:37987 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/87fb3d252950:0, corePoolSize=1, maxPoolSize=1 2024-11-18T07:52:05,512 DEBUG [RS:0;87fb3d252950:37987 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/87fb3d252950:0, corePoolSize=1, maxPoolSize=1 2024-11-18T07:52:05,513 DEBUG [RS:0;87fb3d252950:37987 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/87fb3d252950:0, corePoolSize=3, maxPoolSize=3 2024-11-18T07:52:05,513 DEBUG [RS:0;87fb3d252950:37987 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0, corePoolSize=3, maxPoolSize=3 2024-11-18T07:52:05,519 INFO [RS:0;87fb3d252950:37987 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:05,519 INFO [RS:0;87fb3d252950:37987 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:05,519 INFO [RS:0;87fb3d252950:37987 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:05,520 INFO [RS:0;87fb3d252950:37987 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:05,520 INFO [RS:0;87fb3d252950:37987 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:05,520 INFO [RS:0;87fb3d252950:37987 {}] hbase.ChoreService(168): Chore ScheduledChore name=87fb3d252950,37987,1731916323128-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T07:52:05,547 INFO [RS:0;87fb3d252950:37987 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T07:52:05,549 INFO [RS:0;87fb3d252950:37987 {}] hbase.ChoreService(168): Chore ScheduledChore name=87fb3d252950,37987,1731916323128-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:05,550 INFO [RS:0;87fb3d252950:37987 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:05,550 INFO [RS:0;87fb3d252950:37987 {}] regionserver.Replication(171): 87fb3d252950,37987,1731916323128 started 2024-11-18T07:52:05,576 INFO [RS:0;87fb3d252950:37987 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:05,576 INFO [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(1482): Serving as 87fb3d252950,37987,1731916323128, RpcServer on 87fb3d252950/172.17.0.2:37987, sessionid=0x1014e3765000001 2024-11-18T07:52:05,577 DEBUG [RS:0;87fb3d252950:37987 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T07:52:05,577 DEBUG [RS:0;87fb3d252950:37987 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 87fb3d252950,37987,1731916323128 2024-11-18T07:52:05,578 DEBUG [RS:0;87fb3d252950:37987 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '87fb3d252950,37987,1731916323128' 2024-11-18T07:52:05,578 DEBUG [RS:0;87fb3d252950:37987 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T07:52:05,579 DEBUG [RS:0;87fb3d252950:37987 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T07:52:05,580 DEBUG [RS:0;87fb3d252950:37987 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T07:52:05,580 DEBUG [RS:0;87fb3d252950:37987 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T07:52:05,580 DEBUG [RS:0;87fb3d252950:37987 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 87fb3d252950,37987,1731916323128 2024-11-18T07:52:05,580 DEBUG [RS:0;87fb3d252950:37987 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '87fb3d252950,37987,1731916323128' 2024-11-18T07:52:05,581 DEBUG [RS:0;87fb3d252950:37987 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T07:52:05,581 DEBUG [RS:0;87fb3d252950:37987 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T07:52:05,582 DEBUG [RS:0;87fb3d252950:37987 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T07:52:05,582 INFO [RS:0;87fb3d252950:37987 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T07:52:05,582 INFO [RS:0;87fb3d252950:37987 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T07:52:05,626 WARN [87fb3d252950:42633 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T07:52:05,689 INFO [RS:0;87fb3d252950:37987 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T07:52:05,693 INFO [RS:0;87fb3d252950:37987 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=87fb3d252950%2C37987%2C1731916323128, suffix=, logDir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/WALs/87fb3d252950,37987,1731916323128, archiveDir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/oldWALs, maxLogs=32 2024-11-18T07:52:05,716 DEBUG [RS:0;87fb3d252950:37987 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/WALs/87fb3d252950,37987,1731916323128/87fb3d252950%2C37987%2C1731916323128.1731916325696, exclude list is [], retry=0 2024-11-18T07:52:05,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40631,DS-54035976-ce14-4282-869f-c28be41b513f,DISK] 2024-11-18T07:52:05,725 INFO [RS:0;87fb3d252950:37987 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/WALs/87fb3d252950,37987,1731916323128/87fb3d252950%2C37987%2C1731916323128.1731916325696 2024-11-18T07:52:05,726 DEBUG [RS:0;87fb3d252950:37987 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39457:39457)] 2024-11-18T07:52:05,878 DEBUG [87fb3d252950:42633 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-18T07:52:05,889 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:52:05,897 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87fb3d252950,37987,1731916323128, state=OPENING 2024-11-18T07:52:05,923 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T07:52:05,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T07:52:05,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T07:52:05,933 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T07:52:05,934 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T07:52:05,935 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T07:52:05,938 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:52:06,115 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T07:52:06,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38803, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T07:52:06,133 INFO [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T07:52:06,133 INFO [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T07:52:06,134 INFO [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T07:52:06,138 INFO [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=87fb3d252950%2C37987%2C1731916323128.meta, suffix=.meta, logDir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/WALs/87fb3d252950,37987,1731916323128, archiveDir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/oldWALs, maxLogs=32 2024-11-18T07:52:06,162 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/WALs/87fb3d252950,37987,1731916323128/87fb3d252950%2C37987%2C1731916323128.meta.1731916326141.meta, exclude list is [], retry=0 2024-11-18T07:52:06,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40631,DS-54035976-ce14-4282-869f-c28be41b513f,DISK] 2024-11-18T07:52:06,170 INFO [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/WALs/87fb3d252950,37987,1731916323128/87fb3d252950%2C37987%2C1731916323128.meta.1731916326141.meta 2024-11-18T07:52:06,171 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39457:39457)] 2024-11-18T07:52:06,171 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T07:52:06,174 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T07:52:06,177 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T07:52:06,183 INFO [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T07:52:06,189 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T07:52:06,190 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:52:06,190 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T07:52:06,191 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T07:52:06,195 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T07:52:06,197 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T07:52:06,197 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:06,199 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T07:52:06,199 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T07:52:06,201 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T07:52:06,202 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:06,203 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T07:52:06,203 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T07:52:06,205 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T07:52:06,206 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:06,207 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T07:52:06,207 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T07:52:06,210 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T07:52:06,210 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:06,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T07:52:06,211 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T07:52:06,213 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740 2024-11-18T07:52:06,218 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740 2024-11-18T07:52:06,221 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T07:52:06,221 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T07:52:06,222 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T07:52:06,227 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T07:52:06,229 INFO [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64279135, jitterRate=-0.042166247963905334}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T07:52:06,230 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T07:52:06,232 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731916326191Writing region info on filesystem at 1731916326191Initializing all the Stores at 1731916326194 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731916326194Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731916326194Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916326194Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731916326194Cleaning up temporary data from old regions at 1731916326221 (+27 ms)Running coprocessor post-open hooks at 1731916326230 (+9 ms)Region opened successfully at 1731916326232 (+2 ms) 2024-11-18T07:52:06,242 INFO [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731916326107 2024-11-18T07:52:06,257 DEBUG [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T07:52:06,257 INFO [RS_OPEN_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T07:52:06,260 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:52:06,263 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87fb3d252950,37987,1731916323128, state=OPEN 2024-11-18T07:52:06,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T07:52:06,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T07:52:06,328 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T07:52:06,328 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T07:52:06,329 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:06,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T07:52:06,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87fb3d252950,37987,1731916323128 in 392 msec 2024-11-18T07:52:06,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T07:52:06,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 877 msec 2024-11-18T07:52:06,349 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T07:52:06,349 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T07:52:06,376 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:06,378 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:06,404 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:06,407 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57411, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:06,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1980 sec 2024-11-18T07:52:06,436 INFO [master/87fb3d252950:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731916326436, completionTime=-1 2024-11-18T07:52:06,439 INFO [master/87fb3d252950:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-18T07:52:06,441 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T07:52:06,472 INFO [master/87fb3d252950:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-18T07:52:06,472 INFO [master/87fb3d252950:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731916386472 2024-11-18T07:52:06,472 INFO [master/87fb3d252950:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731916446472 2024-11-18T07:52:06,472 INFO [master/87fb3d252950:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-11-18T07:52:06,475 INFO [master/87fb3d252950:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87fb3d252950,42633,1731916322278-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:06,476 INFO [master/87fb3d252950:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87fb3d252950,42633,1731916322278-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:06,476 INFO [master/87fb3d252950:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87fb3d252950,42633,1731916322278-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:06,478 INFO [master/87fb3d252950:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-87fb3d252950:42633, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:06,478 INFO [master/87fb3d252950:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:06,479 INFO [master/87fb3d252950:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:06,485 DEBUG [master/87fb3d252950:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T07:52:06,516 INFO [master/87fb3d252950:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.166sec 2024-11-18T07:52:06,517 INFO [master/87fb3d252950:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T07:52:06,519 INFO [master/87fb3d252950:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T07:52:06,520 INFO [master/87fb3d252950:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T07:52:06,521 INFO [master/87fb3d252950:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T07:52:06,521 INFO [master/87fb3d252950:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T07:52:06,523 INFO [master/87fb3d252950:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87fb3d252950,42633,1731916322278-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T07:52:06,523 INFO [master/87fb3d252950:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87fb3d252950,42633,1731916322278-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T07:52:06,533 DEBUG [master/87fb3d252950:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T07:52:06,534 INFO [master/87fb3d252950:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T07:52:06,535 INFO [master/87fb3d252950:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87fb3d252950,42633,1731916322278-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T07:52:06,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d237762, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:06,570 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T07:52:06,570 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T07:52:06,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:06,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:06,585 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:06,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:06,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a872003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:06,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:06,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:06,599 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:06,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6db57897, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:06,603 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:06,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:06,635 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=87fb3d252950,42633,1731916322278 2024-11-18T07:52:06,649 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=497, ProcessCount=11, AvailableMemoryMB=980 2024-11-18T07:52:06,657 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T07:52:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 87fb3d252950,42633,1731916322278 2024-11-18T07:52:06,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56ef0928 2024-11-18T07:52:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T07:52:06,716 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T07:52:06,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T07:52:06,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T07:52:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T07:52:06,756 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T07:52:06,760 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-18T07:52:06,761 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:06,765 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T07:52:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T07:52:06,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741835_1011 (size=1061) 2024-11-18T07:52:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T07:52:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T07:52:07,183 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3 2024-11-18T07:52:07,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741836_1012 (size=53) 2024-11-18T07:52:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T07:52:07,594 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:52:07,594 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1563b121b61f4db988d82be75c71ace9, disabling compactions & flushes 2024-11-18T07:52:07,594 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:07,594 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:07,594 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. after waiting 0 ms 2024-11-18T07:52:07,594 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:07,595 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:07,595 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1563b121b61f4db988d82be75c71ace9: Waiting for close lock at 1731916327594Disabling compacts and flushes for region at 1731916327594Disabling writes for close at 1731916327594Writing region close event to WAL at 1731916327595 (+1 ms)Closed at 1731916327595 2024-11-18T07:52:07,598 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T07:52:07,605 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731916327598"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731916327598"}]},"ts":"1731916327598"} 2024-11-18T07:52:07,611 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T07:52:07,613 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T07:52:07,617 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916327614"}]},"ts":"1731916327614"} 2024-11-18T07:52:07,623 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T07:52:07,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1563b121b61f4db988d82be75c71ace9, ASSIGN}] 2024-11-18T07:52:07,628 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1563b121b61f4db988d82be75c71ace9, ASSIGN 2024-11-18T07:52:07,631 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1563b121b61f4db988d82be75c71ace9, ASSIGN; state=OFFLINE, location=87fb3d252950,37987,1731916323128; forceNewPlan=false, retain=false 2024-11-18T07:52:07,783 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1563b121b61f4db988d82be75c71ace9, regionState=OPENING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:52:07,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1563b121b61f4db988d82be75c71ace9, ASSIGN because future has completed 2024-11-18T07:52:07,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:52:07,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T07:52:07,954 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:07,954 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} 2024-11-18T07:52:07,955 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:07,955 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:52:07,955 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:07,955 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:07,958 INFO [StoreOpener-1563b121b61f4db988d82be75c71ace9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:07,962 INFO [StoreOpener-1563b121b61f4db988d82be75c71ace9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:52:07,962 INFO [StoreOpener-1563b121b61f4db988d82be75c71ace9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1563b121b61f4db988d82be75c71ace9 columnFamilyName A 2024-11-18T07:52:07,962 DEBUG [StoreOpener-1563b121b61f4db988d82be75c71ace9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:07,964 INFO [StoreOpener-1563b121b61f4db988d82be75c71ace9-1 {}] regionserver.HStore(327): Store=1563b121b61f4db988d82be75c71ace9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:52:07,964 INFO [StoreOpener-1563b121b61f4db988d82be75c71ace9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:07,966 INFO [StoreOpener-1563b121b61f4db988d82be75c71ace9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:52:07,967 INFO [StoreOpener-1563b121b61f4db988d82be75c71ace9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1563b121b61f4db988d82be75c71ace9 columnFamilyName B 2024-11-18T07:52:07,967 DEBUG [StoreOpener-1563b121b61f4db988d82be75c71ace9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:07,968 INFO [StoreOpener-1563b121b61f4db988d82be75c71ace9-1 {}] regionserver.HStore(327): Store=1563b121b61f4db988d82be75c71ace9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:52:07,968 INFO [StoreOpener-1563b121b61f4db988d82be75c71ace9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:07,971 INFO [StoreOpener-1563b121b61f4db988d82be75c71ace9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:52:07,971 INFO [StoreOpener-1563b121b61f4db988d82be75c71ace9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1563b121b61f4db988d82be75c71ace9 columnFamilyName C 2024-11-18T07:52:07,971 DEBUG [StoreOpener-1563b121b61f4db988d82be75c71ace9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:07,972 INFO [StoreOpener-1563b121b61f4db988d82be75c71ace9-1 {}] regionserver.HStore(327): Store=1563b121b61f4db988d82be75c71ace9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:52:07,973 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:07,973 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:07,975 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:07,976 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:07,980 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:07,980 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:07,982 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T07:52:07,986 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:07,990 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T07:52:07,991 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1563b121b61f4db988d82be75c71ace9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70085249, jitterRate=0.044351592659950256}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T07:52:07,991 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:07,992 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1563b121b61f4db988d82be75c71ace9: Running coprocessor pre-open hook at 1731916327955Writing region info on filesystem at 1731916327956 (+1 ms)Initializing all the Stores at 1731916327957 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916327957Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916327958 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916327958Cleaning up temporary data from old regions at 1731916327980 (+22 ms)Running coprocessor post-open hooks at 1731916327991 (+11 ms)Region opened successfully at 1731916327992 (+1 ms) 2024-11-18T07:52:07,994 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., pid=6, masterSystemTime=1731916327946 2024-11-18T07:52:07,999 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:07,999 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:08,000 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1563b121b61f4db988d82be75c71ace9, regionState=OPEN, openSeqNum=2, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:52:08,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:52:08,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T07:52:08,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 in 217 msec 2024-11-18T07:52:08,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T07:52:08,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1563b121b61f4db988d82be75c71ace9, ASSIGN in 389 msec 2024-11-18T07:52:08,022 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T07:52:08,022 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916328022"}]},"ts":"1731916328022"} 2024-11-18T07:52:08,026 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T07:52:08,029 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T07:52:08,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2990 sec 2024-11-18T07:52:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T07:52:08,930 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:08,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@245cc74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:08,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:08,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:08,938 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:08,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:08,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5583d40c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:08,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:08,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:08,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:08,941 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:08,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@665358ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:08,944 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:08,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:08,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:08,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:08,951 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T07:52:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 87fb3d252950,42633,1731916322278 2024-11-18T07:52:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@32b26fa3 2024-11-18T07:52:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T07:52:08,960 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T07:52:08,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a519624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:08,965 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:08,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:08,968 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149abd79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:08,980 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:08,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aa662a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:08,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:08,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:08,985 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:08,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:08,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:08,990 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:08,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a38e5f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:08,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:08,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:08,995 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:08,999 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658d066f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:09,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:09,003 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa64271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:09,006 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76e341cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b988aa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:09,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:09,009 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f695ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:09,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:09,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:09,012 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25318c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32cda1eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:09,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:09,016 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:09,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3c1f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:09,019 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e1b379, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,021 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d3b492, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:09,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:09,022 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:09,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:09,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec64ade, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:09,025 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:09,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e58d725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27451c77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:09,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:09,028 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c24f6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:09,031 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:09,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e2fecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59639c43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:09,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:09,035 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70fb5a22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:09,037 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:09,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f696c27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:09,044 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:09,044 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:09,045 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:09,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:09,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:09,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:09,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:09,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:09,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:09,049 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:09,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:09,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:09,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:09,053 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:09,053 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:09,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:52:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:52:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:52:09,058 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:09,059 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:52:09,060 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:09,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:52:09,067 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:09,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:52:09,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-18T07:52:09,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:09,072 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:52:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T07:52:09,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:09,074 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:52:09,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:52:09,076 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:09,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:52:09,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:09,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:09,079 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:09,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:09,083 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:09,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:09,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:09,093 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:09,094 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:09,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:52:09,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:52:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:52:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T07:52:09,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T07:52:09,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:09,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T07:52:09,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:09,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:09,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:09,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:09,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:09,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/d8fdf21ee5e34c0eb4ce8f756570afbf is 50, key is test_row_0/A:col10/1731916329168/Put/seqid=0 2024-11-18T07:52:09,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741837_1013 (size=16681) 2024-11-18T07:52:09,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/d8fdf21ee5e34c0eb4ce8f756570afbf 2024-11-18T07:52:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T07:52:09,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916389322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:09,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T07:52:09,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916389335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:09,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:09,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916389392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:09,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:09,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:09,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:09,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:09,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916389418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:09,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916389343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:09,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:09,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:09,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:09,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916389590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:09,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916389592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:09,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916389600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:09,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:09,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916389601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916389601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:09,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/4fa7023dc491425fbe768a705799c475 is 50, key is test_row_0/B:col10/1731916329168/Put/seqid=0 2024-11-18T07:52:09,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:09,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T07:52:09,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:09,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:09,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:09,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:09,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:09,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741838_1014 (size=12001) 2024-11-18T07:52:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T07:52:09,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T07:52:09,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:09,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:09,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:09,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:09,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:09,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916389809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:09,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916389810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:09,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:09,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916389810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:09,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916389823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:09,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:09,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916389823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:09,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T07:52:09,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:09,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:09,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:09,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:09,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/4fa7023dc491425fbe768a705799c475 2024-11-18T07:52:10,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T07:52:10,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:10,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:10,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:10,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916390131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:10,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/e472e44ddac34fb8941ed470ae4e0549 is 50, key is test_row_0/C:col10/1731916329168/Put/seqid=0 2024-11-18T07:52:10,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916390135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:10,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916390135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:10,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916390140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:10,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916390140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:10,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741839_1015 (size=12001) 2024-11-18T07:52:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T07:52:10,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T07:52:10,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:10,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:10,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:10,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T07:52:10,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:10,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:10,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:10,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/e472e44ddac34fb8941ed470ae4e0549 2024-11-18T07:52:10,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T07:52:10,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:10,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:10,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:10,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/d8fdf21ee5e34c0eb4ce8f756570afbf as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/d8fdf21ee5e34c0eb4ce8f756570afbf 2024-11-18T07:52:10,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/d8fdf21ee5e34c0eb4ce8f756570afbf, entries=250, sequenceid=14, filesize=16.3 K 2024-11-18T07:52:10,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/4fa7023dc491425fbe768a705799c475 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4fa7023dc491425fbe768a705799c475 2024-11-18T07:52:10,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4fa7023dc491425fbe768a705799c475, entries=150, sequenceid=14, filesize=11.7 K 2024-11-18T07:52:10,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/e472e44ddac34fb8941ed470ae4e0549 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e472e44ddac34fb8941ed470ae4e0549 2024-11-18T07:52:10,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e472e44ddac34fb8941ed470ae4e0549, entries=150, sequenceid=14, filesize=11.7 K 2024-11-18T07:52:10,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1563b121b61f4db988d82be75c71ace9 in 1504ms, sequenceid=14, compaction requested=false 2024-11-18T07:52:10,684 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T07:52:10,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:10,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=114.05 KB heapSize=339.89 KB 2024-11-18T07:52:10,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T07:52:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:10,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:10,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:10,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:10,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:10,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:10,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:10,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/f6802fbd7efe4b8e8e3539ce81b1d30f is 50, key is test_row_0/A:col10/1731916330721/Put/seqid=0 2024-11-18T07:52:10,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741840_1016 (size=12001) 2024-11-18T07:52:10,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/f6802fbd7efe4b8e8e3539ce81b1d30f 2024-11-18T07:52:10,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916390803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:10,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:10,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916390833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916390845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:10,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T07:52:10,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:10,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:10,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:10,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:10,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:10,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916390882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:10,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916390885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:10,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/7868fdc0fb6a4bf69a0cb47273a02a6a is 50, key is test_row_0/B:col10/1731916330721/Put/seqid=0 2024-11-18T07:52:10,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741841_1017 (size=12001) 2024-11-18T07:52:10,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/7868fdc0fb6a4bf69a0cb47273a02a6a 2024-11-18T07:52:11,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/0525911b865e4e3cbbb0fb5ab38f1d1e is 50, key is test_row_0/C:col10/1731916330721/Put/seqid=0 2024-11-18T07:52:11,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916390992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916390999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916391000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916391041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T07:52:11,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:11,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:11,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:11,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:11,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:11,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741842_1018 (size=12001) 2024-11-18T07:52:11,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/0525911b865e4e3cbbb0fb5ab38f1d1e 2024-11-18T07:52:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916391045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/f6802fbd7efe4b8e8e3539ce81b1d30f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/f6802fbd7efe4b8e8e3539ce81b1d30f 2024-11-18T07:52:11,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/f6802fbd7efe4b8e8e3539ce81b1d30f, entries=150, sequenceid=36, filesize=11.7 K 2024-11-18T07:52:11,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/7868fdc0fb6a4bf69a0cb47273a02a6a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/7868fdc0fb6a4bf69a0cb47273a02a6a 2024-11-18T07:52:11,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/7868fdc0fb6a4bf69a0cb47273a02a6a, entries=150, sequenceid=36, filesize=11.7 K 2024-11-18T07:52:11,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/0525911b865e4e3cbbb0fb5ab38f1d1e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/0525911b865e4e3cbbb0fb5ab38f1d1e 2024-11-18T07:52:11,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/0525911b865e4e3cbbb0fb5ab38f1d1e, entries=150, sequenceid=36, filesize=11.7 K 2024-11-18T07:52:11,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 1563b121b61f4db988d82be75c71ace9 in 403ms, sequenceid=36, compaction requested=false 2024-11-18T07:52:11,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:11,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T07:52:11,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:11,216 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:52:11,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T07:52:11,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:11,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:11,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:11,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:11,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:11,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:11,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/a72c372715124beb943d14fb7a843e8c is 50, key is test_row_0/A:col10/1731916330865/Put/seqid=0 2024-11-18T07:52:11,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741843_1019 (size=12001) 2024-11-18T07:52:11,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916331309, max timestamp=1731916331383Num uniques 859; 2024-11-18T07:52:11,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:52:11,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916331309, max timestamp=1731916331386Num uniques 859; 2024-11-18T07:52:11,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:52:11,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916331309, max timestamp=1731916331383Num uniques 859; 2024-11-18T07:52:11,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:52:11,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916391480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916391480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916391480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916391485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916391505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916391626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916391626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916391629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916391630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916391639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,720 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/a72c372715124beb943d14fb7a843e8c 2024-11-18T07:52:11,728 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T07:52:11,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/79d13da74e824c8db7ebc0a76ce18981 is 50, key is test_row_0/B:col10/1731916330865/Put/seqid=0 2024-11-18T07:52:11,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741844_1020 (size=12001) 2024-11-18T07:52:11,821 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/79d13da74e824c8db7ebc0a76ce18981 2024-11-18T07:52:11,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916391838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916391839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916391848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916391870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/74750b4f07c342cfafdc07cb3d20ead7 is 50, key is test_row_0/C:col10/1731916330865/Put/seqid=0 2024-11-18T07:52:11,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916391871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:11,908 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T07:52:11,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741845_1021 (size=12001) 2024-11-18T07:52:11,915 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/74750b4f07c342cfafdc07cb3d20ead7 2024-11-18T07:52:11,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/a72c372715124beb943d14fb7a843e8c as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a72c372715124beb943d14fb7a843e8c 2024-11-18T07:52:11,950 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a72c372715124beb943d14fb7a843e8c, entries=150, sequenceid=46, filesize=11.7 K 2024-11-18T07:52:11,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/79d13da74e824c8db7ebc0a76ce18981 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/79d13da74e824c8db7ebc0a76ce18981 2024-11-18T07:52:11,971 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/79d13da74e824c8db7ebc0a76ce18981, entries=150, sequenceid=46, filesize=11.7 K 2024-11-18T07:52:11,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/74750b4f07c342cfafdc07cb3d20ead7 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/74750b4f07c342cfafdc07cb3d20ead7 2024-11-18T07:52:11,991 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/74750b4f07c342cfafdc07cb3d20ead7, entries=150, sequenceid=46, filesize=11.7 K 2024-11-18T07:52:11,997 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1563b121b61f4db988d82be75c71ace9 in 781ms, sequenceid=46, compaction requested=true 2024-11-18T07:52:11,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:11,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:11,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-18T07:52:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-18T07:52:12,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T07:52:12,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9240 sec 2024-11-18T07:52:12,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.9500 sec 2024-11-18T07:52:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:12,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-18T07:52:12,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:12,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:12,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:12,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:12,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:12,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:12,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/71c5c4282f894a96b37897dd340a3428 is 50, key is test_row_0/A:col10/1731916331500/Put/seqid=0 2024-11-18T07:52:12,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916392217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916392222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916392225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916392221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916392240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741846_1022 (size=14341) 2024-11-18T07:52:12,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/71c5c4282f894a96b37897dd340a3428 2024-11-18T07:52:12,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/288c55225a5949a19e99516615fc3df3 is 50, key is test_row_0/B:col10/1731916331500/Put/seqid=0 2024-11-18T07:52:12,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741847_1023 (size=12001) 2024-11-18T07:52:12,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/288c55225a5949a19e99516615fc3df3 2024-11-18T07:52:12,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916392359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916392360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916392360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916392360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916392365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/f9539de564ce46fe852c2a871d245138 is 50, key is test_row_0/C:col10/1731916331500/Put/seqid=0 2024-11-18T07:52:12,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741848_1024 (size=12001) 2024-11-18T07:52:12,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916392578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916392589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916392589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916392589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916392599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,725 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-18T07:52:12,726 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-18T07:52:12,728 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T07:52:12,728 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-18T07:52:12,728 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-18T07:52:12,729 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-18T07:52:12,730 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T07:52:12,730 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-18T07:52:12,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/f9539de564ce46fe852c2a871d245138 2024-11-18T07:52:12,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/71c5c4282f894a96b37897dd340a3428 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/71c5c4282f894a96b37897dd340a3428 2024-11-18T07:52:12,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916392891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916392900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/71c5c4282f894a96b37897dd340a3428, entries=200, sequenceid=83, filesize=14.0 K 2024-11-18T07:52:12,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916392903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916392910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/288c55225a5949a19e99516615fc3df3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/288c55225a5949a19e99516615fc3df3 2024-11-18T07:52:12,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/288c55225a5949a19e99516615fc3df3, entries=150, sequenceid=83, filesize=11.7 K 2024-11-18T07:52:12,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/f9539de564ce46fe852c2a871d245138 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/f9539de564ce46fe852c2a871d245138 2024-11-18T07:52:12,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916392918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:12,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/f9539de564ce46fe852c2a871d245138, entries=150, sequenceid=83, filesize=11.7 K 2024-11-18T07:52:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:12,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 1563b121b61f4db988d82be75c71ace9 in 768ms, sequenceid=83, compaction requested=true 2024-11-18T07:52:12,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:12,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:12,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:12,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:12,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:12,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:12,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:52:12,980 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:12,981 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:12,987 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:12,989 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/B is initiating minor compaction (all files) 2024-11-18T07:52:12,990 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/B in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:12,991 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4fa7023dc491425fbe768a705799c475, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/7868fdc0fb6a4bf69a0cb47273a02a6a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/79d13da74e824c8db7ebc0a76ce18981, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/288c55225a5949a19e99516615fc3df3] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=46.9 K 2024-11-18T07:52:12,993 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fa7023dc491425fbe768a705799c475, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731916329155 2024-11-18T07:52:12,995 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7868fdc0fb6a4bf69a0cb47273a02a6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731916329320 2024-11-18T07:52:12,996 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 79d13da74e824c8db7ebc0a76ce18981, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731916330834 2024-11-18T07:52:12,997 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 288c55225a5949a19e99516615fc3df3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731916331468 2024-11-18T07:52:13,002 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:13,002 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/A is initiating minor compaction (all files) 2024-11-18T07:52:13,003 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/A in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:13,003 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/d8fdf21ee5e34c0eb4ce8f756570afbf, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/f6802fbd7efe4b8e8e3539ce81b1d30f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a72c372715124beb943d14fb7a843e8c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/71c5c4282f894a96b37897dd340a3428] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=53.7 K 2024-11-18T07:52:13,006 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8fdf21ee5e34c0eb4ce8f756570afbf, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731916329123 2024-11-18T07:52:13,009 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6802fbd7efe4b8e8e3539ce81b1d30f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731916329320 2024-11-18T07:52:13,013 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a72c372715124beb943d14fb7a843e8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731916330834 2024-11-18T07:52:13,019 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71c5c4282f894a96b37897dd340a3428, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731916331468 2024-11-18T07:52:13,070 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#B#compaction#12 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:13,073 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/d2ca4aa8c8c04238a7774551ae0cfc72 is 50, key is test_row_0/B:col10/1731916331500/Put/seqid=0 2024-11-18T07:52:13,111 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#A#compaction#13 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:13,112 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/5945cc1f25a24d059a00009476b0d1c1 is 50, key is test_row_0/A:col10/1731916331500/Put/seqid=0 2024-11-18T07:52:13,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741849_1025 (size=12139) 2024-11-18T07:52:13,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741850_1026 (size=12139) 2024-11-18T07:52:13,210 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/5945cc1f25a24d059a00009476b0d1c1 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/5945cc1f25a24d059a00009476b0d1c1 2024-11-18T07:52:13,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T07:52:13,229 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:13,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:52:13,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-18T07:52:13,234 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:52:13,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T07:52:13,236 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:52:13,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:52:13,264 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/A of 1563b121b61f4db988d82be75c71ace9 into 5945cc1f25a24d059a00009476b0d1c1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:13,264 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:13,267 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/A, priority=12, startTime=1731916332955; duration=0sec 2024-11-18T07:52:13,268 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:13,268 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:A 2024-11-18T07:52:13,268 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:13,271 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:13,271 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/C is initiating minor compaction (all files) 2024-11-18T07:52:13,272 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/C in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:13,272 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e472e44ddac34fb8941ed470ae4e0549, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/0525911b865e4e3cbbb0fb5ab38f1d1e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/74750b4f07c342cfafdc07cb3d20ead7, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/f9539de564ce46fe852c2a871d245138] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=46.9 K 2024-11-18T07:52:13,273 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e472e44ddac34fb8941ed470ae4e0549, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731916329155 2024-11-18T07:52:13,274 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0525911b865e4e3cbbb0fb5ab38f1d1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731916329320 2024-11-18T07:52:13,274 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74750b4f07c342cfafdc07cb3d20ead7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731916330834 2024-11-18T07:52:13,275 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9539de564ce46fe852c2a871d245138, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731916331468 2024-11-18T07:52:13,317 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#C#compaction#14 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:13,318 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/05f6d0c059da4cb8afc73f60c2d78498 is 50, key is test_row_0/C:col10/1731916331500/Put/seqid=0 2024-11-18T07:52:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T07:52:13,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741851_1027 (size=12139) 2024-11-18T07:52:13,383 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/05f6d0c059da4cb8afc73f60c2d78498 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/05f6d0c059da4cb8afc73f60c2d78498 2024-11-18T07:52:13,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T07:52:13,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:13,393 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-18T07:52:13,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:13,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:13,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:13,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:13,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:13,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:13,406 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/C of 1563b121b61f4db988d82be75c71ace9 into 05f6d0c059da4cb8afc73f60c2d78498(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:13,406 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:13,406 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/C, priority=12, startTime=1731916332969; duration=0sec 2024-11-18T07:52:13,406 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:13,406 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:C 2024-11-18T07:52:13,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/afd33fc1422d468d9502672e80e456e5 is 50, key is test_row_0/A:col10/1731916332235/Put/seqid=0 2024-11-18T07:52:13,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:13,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:13,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741852_1028 (size=9657) 2024-11-18T07:52:13,537 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/d2ca4aa8c8c04238a7774551ae0cfc72 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/d2ca4aa8c8c04238a7774551ae0cfc72 2024-11-18T07:52:13,553 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/B of 1563b121b61f4db988d82be75c71ace9 into d2ca4aa8c8c04238a7774551ae0cfc72(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:13,553 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:13,553 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/B, priority=12, startTime=1731916332968; duration=0sec 2024-11-18T07:52:13,554 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:13,554 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:B 2024-11-18T07:52:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T07:52:13,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731916333461, max timestamp=1731916333556Num uniques 859; 2024-11-18T07:52:13,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:52:13,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731916333461, max timestamp=1731916333556Num uniques 859; 2024-11-18T07:52:13,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:52:13,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731916333461, max timestamp=1731916333556Num uniques 859; 2024-11-18T07:52:13,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:52:13,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:13,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916393631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916393627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:13,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:13,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916393634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:13,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:13,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:13,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916393649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916393649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:13,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:13,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:13,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916393759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:13,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:13,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916393759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:13,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916393759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:13,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:13,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:13,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916393779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:13,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:13,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916393779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:13,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T07:52:13,884 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/afd33fc1422d468d9502672e80e456e5 2024-11-18T07:52:13,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/4784cc50c6304526a31fa41c3bc15099 is 50, key is test_row_0/B:col10/1731916332235/Put/seqid=0 2024-11-18T07:52:13,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741853_1029 (size=9657) 2024-11-18T07:52:13,974 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/4784cc50c6304526a31fa41c3bc15099 2024-11-18T07:52:13,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916393980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916393980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:13,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:13,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916393980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:13,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:13,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:14,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916393999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:14,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:14,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/d869bf3590b74cef9c8ef0c427ab41ce is 50, key is test_row_0/C:col10/1731916332235/Put/seqid=0 2024-11-18T07:52:14,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916394008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:14,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741854_1030 (size=9657) 2024-11-18T07:52:14,040 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/d869bf3590b74cef9c8ef0c427ab41ce 2024-11-18T07:52:14,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/afd33fc1422d468d9502672e80e456e5 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/afd33fc1422d468d9502672e80e456e5 2024-11-18T07:52:14,083 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/afd33fc1422d468d9502672e80e456e5, entries=100, sequenceid=93, filesize=9.4 K 2024-11-18T07:52:14,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/4784cc50c6304526a31fa41c3bc15099 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4784cc50c6304526a31fa41c3bc15099 2024-11-18T07:52:14,097 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4784cc50c6304526a31fa41c3bc15099, entries=100, sequenceid=93, filesize=9.4 K 2024-11-18T07:52:14,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/d869bf3590b74cef9c8ef0c427ab41ce as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/d869bf3590b74cef9c8ef0c427ab41ce 2024-11-18T07:52:14,118 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/d869bf3590b74cef9c8ef0c427ab41ce, entries=100, sequenceid=93, filesize=9.4 K 2024-11-18T07:52:14,122 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for 1563b121b61f4db988d82be75c71ace9 in 727ms, sequenceid=93, compaction requested=false 2024-11-18T07:52:14,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:14,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:14,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-18T07:52:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-18T07:52:14,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-18T07:52:14,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 888 msec 2024-11-18T07:52:14,133 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 900 msec 2024-11-18T07:52:14,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731916333617, max timestamp=1731916334305Num uniques 1118; 2024-11-18T07:52:14,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731916333617, max timestamp=1731916334305Num uniques 1118; 2024-11-18T07:52:14,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-18T07:52:14,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-18T07:52:14,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=76.04 KB heapSize=208.88 KB 2024-11-18T07:52:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:14,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731916333617, max timestamp=1731916334301Num uniques 1118; 2024-11-18T07:52:14,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-18T07:52:14,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:14,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:14,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:14,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:14,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:14,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:14,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/976241ab61854183b1cc17bb55af06ee is 50, key is test_row_0/A:col10/1731916333617/Put/seqid=0 2024-11-18T07:52:14,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741855_1031 (size=12101) 2024-11-18T07:52:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T07:52:14,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/976241ab61854183b1cc17bb55af06ee 2024-11-18T07:52:14,379 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:14,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:52:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-18T07:52:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T07:52:14,384 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:52:14,387 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:52:14,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:52:14,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/acdf303b952b455d9478fea68097782a is 50, key is test_row_0/B:col10/1731916333617/Put/seqid=0 2024-11-18T07:52:14,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916334369, max timestamp=1731916334402Num uniques 859; 2024-11-18T07:52:14,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741856_1032 (size=12101) 2024-11-18T07:52:14,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916334369, max timestamp=1731916334401Num uniques 859; 2024-11-18T07:52:14,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:52:14,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:52:14,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/acdf303b952b455d9478fea68097782a 2024-11-18T07:52:14,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916334369, max timestamp=1731916334401Num uniques 859; 2024-11-18T07:52:14,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:52:14,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/2de040b01684451e8fce6a433efa0b76 is 50, key is test_row_0/C:col10/1731916333617/Put/seqid=0 2024-11-18T07:52:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T07:52:14,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741857_1033 (size=12101) 2024-11-18T07:52:14,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/2de040b01684451e8fce6a433efa0b76 2024-11-18T07:52:14,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731916334458, max timestamp=1731916334519Num uniques 1018; 2024-11-18T07:52:14,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T07:52:14,529 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731916334458, max timestamp=1731916334519Num uniques 1018; 2024-11-18T07:52:14,529 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T07:52:14,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731916334458, max timestamp=1731916334519Num uniques 1018; 2024-11-18T07:52:14,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T07:52:14,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/976241ab61854183b1cc17bb55af06ee as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/976241ab61854183b1cc17bb55af06ee 2024-11-18T07:52:14,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T07:52:14,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:14,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:14,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:14,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:14,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/976241ab61854183b1cc17bb55af06ee, entries=150, sequenceid=134, filesize=11.8 K 2024-11-18T07:52:14,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/acdf303b952b455d9478fea68097782a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/acdf303b952b455d9478fea68097782a 2024-11-18T07:52:14,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/acdf303b952b455d9478fea68097782a, entries=150, sequenceid=134, filesize=11.8 K 2024-11-18T07:52:14,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/2de040b01684451e8fce6a433efa0b76 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/2de040b01684451e8fce6a433efa0b76 2024-11-18T07:52:14,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/2de040b01684451e8fce6a433efa0b76, entries=150, sequenceid=134, filesize=11.8 K 2024-11-18T07:52:14,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=80.51 KB/82440 for 1563b121b61f4db988d82be75c71ace9 in 274ms, sequenceid=134, compaction requested=true 2024-11-18T07:52:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:14,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:14,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:14,594 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:14,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:14,594 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:14,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:14,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:14,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:14,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:14,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=87.22 KB heapSize=244.69 KB 2024-11-18T07:52:14,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:14,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:14,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:14,604 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:14,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:14,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:14,604 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/B is initiating minor compaction (all files) 2024-11-18T07:52:14,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:14,604 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/B in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:14,604 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:14,604 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/A is initiating minor compaction (all files) 2024-11-18T07:52:14,604 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/d2ca4aa8c8c04238a7774551ae0cfc72, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4784cc50c6304526a31fa41c3bc15099, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/acdf303b952b455d9478fea68097782a] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=33.1 K 2024-11-18T07:52:14,604 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/A in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:14,605 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/5945cc1f25a24d059a00009476b0d1c1, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/afd33fc1422d468d9502672e80e456e5, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/976241ab61854183b1cc17bb55af06ee] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=33.1 K 2024-11-18T07:52:14,605 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting d2ca4aa8c8c04238a7774551ae0cfc72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731916331468 2024-11-18T07:52:14,606 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5945cc1f25a24d059a00009476b0d1c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731916331468 2024-11-18T07:52:14,606 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4784cc50c6304526a31fa41c3bc15099, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731916332217 2024-11-18T07:52:14,607 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting afd33fc1422d468d9502672e80e456e5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731916332217 2024-11-18T07:52:14,607 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting acdf303b952b455d9478fea68097782a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731916333617 2024-11-18T07:52:14,607 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 976241ab61854183b1cc17bb55af06ee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731916333617 2024-11-18T07:52:14,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/700b669574cd49519abbf5ae652898de is 50, key is test_row_0/A:col10/1731916334564/Put/seqid=0 2024-11-18T07:52:14,637 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:14,638 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/91a9839e0e524d088b8ca73905d960c0 is 50, key is test_row_0/A:col10/1731916333617/Put/seqid=0 2024-11-18T07:52:14,658 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#B#compaction#23 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:14,660 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/38b86b77727e480a9fc743e40be7ed50 is 50, key is test_row_0/B:col10/1731916333617/Put/seqid=0 2024-11-18T07:52:14,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916394670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:14,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:14,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916394674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:14,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916394679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916394676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:14,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T07:52:14,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741858_1034 (size=12151) 2024-11-18T07:52:14,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T07:52:14,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:14,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:14,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/700b669574cd49519abbf5ae652898de 2024-11-18T07:52:14,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:14,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:14,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916394681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:14,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741859_1035 (size=12341) 2024-11-18T07:52:14,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741860_1036 (size=12341) 2024-11-18T07:52:14,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/5650997ce3e84e71abfbdfb83aaf71cc is 50, key is test_row_0/B:col10/1731916334564/Put/seqid=0 2024-11-18T07:52:14,759 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/38b86b77727e480a9fc743e40be7ed50 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/38b86b77727e480a9fc743e40be7ed50 2024-11-18T07:52:14,777 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/B of 1563b121b61f4db988d82be75c71ace9 into 38b86b77727e480a9fc743e40be7ed50(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:14,777 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:14,777 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/B, priority=13, startTime=1731916334594; duration=0sec 2024-11-18T07:52:14,777 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:14,777 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:B 2024-11-18T07:52:14,778 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:14,780 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:14,780 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/C is initiating minor compaction (all files) 2024-11-18T07:52:14,780 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/C in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:14,780 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/05f6d0c059da4cb8afc73f60c2d78498, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/d869bf3590b74cef9c8ef0c427ab41ce, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/2de040b01684451e8fce6a433efa0b76] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=33.1 K 2024-11-18T07:52:14,782 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 05f6d0c059da4cb8afc73f60c2d78498, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731916331468 2024-11-18T07:52:14,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741861_1037 (size=12151) 2024-11-18T07:52:14,785 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting d869bf3590b74cef9c8ef0c427ab41ce, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731916332217 2024-11-18T07:52:14,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/5650997ce3e84e71abfbdfb83aaf71cc 2024-11-18T07:52:14,794 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2de040b01684451e8fce6a433efa0b76, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731916333617 2024-11-18T07:52:14,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916394798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:14,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916394799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:14,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916394799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:14,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916394799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:14,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/dcfffdc3b5ea4211adc63ee07340d1de is 50, key is test_row_0/C:col10/1731916334564/Put/seqid=0 2024-11-18T07:52:14,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916394820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:14,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T07:52:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:14,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:14,861 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#C#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:14,863 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/c5342828cbee49d89e72d21f234c341d is 50, key is test_row_0/C:col10/1731916333617/Put/seqid=0 2024-11-18T07:52:14,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741862_1038 (size=12151) 2024-11-18T07:52:14,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741863_1039 (size=12341) 2024-11-18T07:52:14,972 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/c5342828cbee49d89e72d21f234c341d as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/c5342828cbee49d89e72d21f234c341d 2024-11-18T07:52:14,990 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/C of 1563b121b61f4db988d82be75c71ace9 into c5342828cbee49d89e72d21f234c341d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:14,990 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:14,990 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/C, priority=13, startTime=1731916334594; duration=0sec 2024-11-18T07:52:14,990 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:14,991 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:C 2024-11-18T07:52:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T07:52:15,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T07:52:15,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:15,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:15,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:15,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:15,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:15,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:15,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:15,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916395023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:15,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:15,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:15,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916395023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:15,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916395020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:15,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:15,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916395023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:15,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916395042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:15,145 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/91a9839e0e524d088b8ca73905d960c0 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/91a9839e0e524d088b8ca73905d960c0 2024-11-18T07:52:15,180 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/A of 1563b121b61f4db988d82be75c71ace9 into 91a9839e0e524d088b8ca73905d960c0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:15,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T07:52:15,180 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:15,180 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/A, priority=13, startTime=1731916334594; duration=0sec 2024-11-18T07:52:15,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:15,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:15,181 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:15,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:15,181 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:A 2024-11-18T07:52:15,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:15,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:15,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/dcfffdc3b5ea4211adc63ee07340d1de 2024-11-18T07:52:15,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/700b669574cd49519abbf5ae652898de as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/700b669574cd49519abbf5ae652898de 2024-11-18T07:52:15,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/700b669574cd49519abbf5ae652898de, entries=150, sequenceid=182, filesize=11.9 K 2024-11-18T07:52:15,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/5650997ce3e84e71abfbdfb83aaf71cc as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/5650997ce3e84e71abfbdfb83aaf71cc 2024-11-18T07:52:15,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T07:52:15,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:15,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:15,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:15,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:15,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:15,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916395348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916395348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:15,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916395348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:15,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916395349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:15,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916395349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:15,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:15,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:15,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/5650997ce3e84e71abfbdfb83aaf71cc, entries=150, sequenceid=182, filesize=11.9 K 2024-11-18T07:52:15,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/dcfffdc3b5ea4211adc63ee07340d1de as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/dcfffdc3b5ea4211adc63ee07340d1de 2024-11-18T07:52:15,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/dcfffdc3b5ea4211adc63ee07340d1de, entries=150, sequenceid=182, filesize=11.9 K 2024-11-18T07:52:15,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~264.56 KB/270912, currentSize=87.22 KB/89310 for 1563b121b61f4db988d82be75c71ace9 in 791ms, sequenceid=182, compaction requested=false 2024-11-18T07:52:15,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:15,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T07:52:15,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:15,503 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T07:52:15,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:15,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:15,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:15,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:15,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:15,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T07:52:15,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/4d95ffe3102049a4801a282b0396fba3 is 50, key is test_row_0/A:col10/1731916334678/Put/seqid=0 2024-11-18T07:52:15,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741864_1040 (size=12151) 2024-11-18T07:52:15,571 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/4d95ffe3102049a4801a282b0396fba3 2024-11-18T07:52:15,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/6322c1c6184142df9bfd05dad2f54a87 is 50, key is test_row_0/B:col10/1731916334678/Put/seqid=0 2024-11-18T07:52:15,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741865_1041 (size=12151) 2024-11-18T07:52:15,661 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/6322c1c6184142df9bfd05dad2f54a87 2024-11-18T07:52:15,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/0aa4a2bfa705488dae1772ad253581aa is 50, key is test_row_0/C:col10/1731916334678/Put/seqid=0 2024-11-18T07:52:15,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741866_1042 (size=12151) 2024-11-18T07:52:15,721 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/0aa4a2bfa705488dae1772ad253581aa 2024-11-18T07:52:15,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/4d95ffe3102049a4801a282b0396fba3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/4d95ffe3102049a4801a282b0396fba3 2024-11-18T07:52:15,749 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/4d95ffe3102049a4801a282b0396fba3, entries=150, sequenceid=201, filesize=11.9 K 2024-11-18T07:52:15,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/6322c1c6184142df9bfd05dad2f54a87 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6322c1c6184142df9bfd05dad2f54a87 2024-11-18T07:52:15,768 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6322c1c6184142df9bfd05dad2f54a87, entries=150, sequenceid=201, filesize=11.9 K 2024-11-18T07:52:15,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/0aa4a2bfa705488dae1772ad253581aa as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/0aa4a2bfa705488dae1772ad253581aa 2024-11-18T07:52:15,790 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/0aa4a2bfa705488dae1772ad253581aa, entries=150, sequenceid=201, filesize=11.9 K 2024-11-18T07:52:15,792 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 1563b121b61f4db988d82be75c71ace9 in 288ms, sequenceid=201, compaction requested=true 2024-11-18T07:52:15,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:15,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:15,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-18T07:52:15,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-18T07:52:15,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-18T07:52:15,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4070 sec 2024-11-18T07:52:15,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.4200 sec 2024-11-18T07:52:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:15,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:52:15,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:15,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:15,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:15,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:15,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:15,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:15,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/81106e9692ef4471bf70adb683c2da59 is 50, key is test_row_0/A:col10/1731916335869/Put/seqid=0 2024-11-18T07:52:15,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741867_1043 (size=12151) 2024-11-18T07:52:15,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/81106e9692ef4471bf70adb683c2da59 2024-11-18T07:52:15,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/6e3ba50558764ca8850dbf10b0e020e0 is 50, key is test_row_0/B:col10/1731916335869/Put/seqid=0 2024-11-18T07:52:15,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731916335945, max timestamp=1731916335987Num uniques 859; 2024-11-18T07:52:15,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:52:15,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731916335945, max timestamp=1731916335987Num uniques 859; 2024-11-18T07:52:15,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:52:15,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731916335945, max timestamp=1731916335989Num uniques 859; 2024-11-18T07:52:15,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:52:15,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741868_1044 (size=12151) 2024-11-18T07:52:16,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916396027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916396029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916396029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916396037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916396054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916396159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916396159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916396169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916396169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916396170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916396378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916396378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916396378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916396380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916396388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/6e3ba50558764ca8850dbf10b0e020e0 2024-11-18T07:52:16,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/7a1fae63db84444b8c107b011aa10a68 is 50, key is test_row_0/C:col10/1731916335869/Put/seqid=0 2024-11-18T07:52:16,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741869_1045 (size=12151) 2024-11-18T07:52:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T07:52:16,529 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:16,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:52:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-18T07:52:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T07:52:16,533 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:52:16,534 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:52:16,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:52:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T07:52:16,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T07:52:16,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:16,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:16,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:16,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:16,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:16,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916396688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916396699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916396699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916396707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:16,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916396709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:16,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T07:52:16,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:16,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:16,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:16,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:16,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T07:52:16,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/7a1fae63db84444b8c107b011aa10a68 2024-11-18T07:52:16,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/81106e9692ef4471bf70adb683c2da59 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/81106e9692ef4471bf70adb683c2da59 2024-11-18T07:52:16,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/81106e9692ef4471bf70adb683c2da59, entries=150, sequenceid=212, filesize=11.9 K 2024-11-18T07:52:16,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/6e3ba50558764ca8850dbf10b0e020e0 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6e3ba50558764ca8850dbf10b0e020e0 2024-11-18T07:52:16,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6e3ba50558764ca8850dbf10b0e020e0, entries=150, sequenceid=212, filesize=11.9 K 2024-11-18T07:52:16,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/7a1fae63db84444b8c107b011aa10a68 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7a1fae63db84444b8c107b011aa10a68 2024-11-18T07:52:16,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7a1fae63db84444b8c107b011aa10a68, entries=150, sequenceid=212, filesize=11.9 K 2024-11-18T07:52:16,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 1563b121b61f4db988d82be75c71ace9 in 1046ms, sequenceid=212, compaction requested=true 2024-11-18T07:52:16,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:16,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:16,939 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:16,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:16,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:16,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:16,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:16,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:52:16,939 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:16,941 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:16,941 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/A is initiating minor compaction (all files) 2024-11-18T07:52:16,942 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/A in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:16,942 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/91a9839e0e524d088b8ca73905d960c0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/700b669574cd49519abbf5ae652898de, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/4d95ffe3102049a4801a282b0396fba3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/81106e9692ef4471bf70adb683c2da59] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=47.7 K 2024-11-18T07:52:16,943 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91a9839e0e524d088b8ca73905d960c0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731916333617 2024-11-18T07:52:16,943 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:16,943 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/B is initiating minor compaction (all files) 2024-11-18T07:52:16,944 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 700b669574cd49519abbf5ae652898de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731916334553 2024-11-18T07:52:16,944 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/B in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:16,944 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/38b86b77727e480a9fc743e40be7ed50, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/5650997ce3e84e71abfbdfb83aaf71cc, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6322c1c6184142df9bfd05dad2f54a87, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6e3ba50558764ca8850dbf10b0e020e0] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=47.7 K 2024-11-18T07:52:16,944 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 38b86b77727e480a9fc743e40be7ed50, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731916333617 2024-11-18T07:52:16,944 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d95ffe3102049a4801a282b0396fba3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731916334653 2024-11-18T07:52:16,945 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5650997ce3e84e71abfbdfb83aaf71cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731916334553 2024-11-18T07:52:16,945 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81106e9692ef4471bf70adb683c2da59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731916335869 2024-11-18T07:52:16,946 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6322c1c6184142df9bfd05dad2f54a87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731916334653 2024-11-18T07:52:16,946 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e3ba50558764ca8850dbf10b0e020e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731916335869 2024-11-18T07:52:16,963 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#A#compaction#33 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:16,964 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/71331fd0dde74b6d8a6d158d62c918c6 is 50, key is test_row_0/A:col10/1731916335869/Put/seqid=0 2024-11-18T07:52:16,966 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#B#compaction#34 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:16,967 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/216fb691347a46459a1536638d3f6b5d is 50, key is test_row_0/B:col10/1731916335869/Put/seqid=0 2024-11-18T07:52:17,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T07:52:17,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741870_1046 (size=12527) 2024-11-18T07:52:17,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:17,019 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-18T07:52:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:17,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741871_1047 (size=12527) 2024-11-18T07:52:17,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/1ab81b0ad45b419cb53cc434b2234d30 is 50, key is test_row_0/A:col10/1731916336034/Put/seqid=0 2024-11-18T07:52:17,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741872_1048 (size=12151) 2024-11-18T07:52:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T07:52:17,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:17,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916397247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916397250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:17,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916397255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:17,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916397274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:17,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916397274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:17,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:17,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916397368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:17,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916397378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916397379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:17,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:17,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916397391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:17,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916397392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:17,430 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/71331fd0dde74b6d8a6d158d62c918c6 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/71331fd0dde74b6d8a6d158d62c918c6 2024-11-18T07:52:17,453 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/216fb691347a46459a1536638d3f6b5d as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/216fb691347a46459a1536638d3f6b5d 2024-11-18T07:52:17,459 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/A of 1563b121b61f4db988d82be75c71ace9 into 71331fd0dde74b6d8a6d158d62c918c6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:17,459 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:17,459 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/A, priority=12, startTime=1731916336938; duration=0sec 2024-11-18T07:52:17,459 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:17,459 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:A 2024-11-18T07:52:17,459 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:17,464 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/B of 1563b121b61f4db988d82be75c71ace9 into 216fb691347a46459a1536638d3f6b5d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:17,465 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:17,465 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/B, priority=12, startTime=1731916336939; duration=0sec 2024-11-18T07:52:17,465 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:17,465 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:B 2024-11-18T07:52:17,469 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:17,469 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/C is initiating minor compaction (all files) 2024-11-18T07:52:17,469 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/C in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:17,469 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/c5342828cbee49d89e72d21f234c341d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/dcfffdc3b5ea4211adc63ee07340d1de, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/0aa4a2bfa705488dae1772ad253581aa, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7a1fae63db84444b8c107b011aa10a68] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=47.7 K 2024-11-18T07:52:17,472 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5342828cbee49d89e72d21f234c341d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731916333617 2024-11-18T07:52:17,473 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcfffdc3b5ea4211adc63ee07340d1de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731916334553 2024-11-18T07:52:17,475 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aa4a2bfa705488dae1772ad253581aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731916334653 2024-11-18T07:52:17,476 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a1fae63db84444b8c107b011aa10a68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731916335869 2024-11-18T07:52:17,494 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/1ab81b0ad45b419cb53cc434b2234d30 2024-11-18T07:52:17,517 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#C#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:17,518 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/e435353b60c64be29f300ab578eb0b6e is 50, key is test_row_0/C:col10/1731916335869/Put/seqid=0 2024-11-18T07:52:17,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/81a6943044e04c2e8a7d1b83a231b936 is 50, key is test_row_0/B:col10/1731916336034/Put/seqid=0 2024-11-18T07:52:17,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916397578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:17,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:17,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741873_1049 (size=12527) 2024-11-18T07:52:17,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916397598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:17,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:17,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741874_1050 (size=12151) 2024-11-18T07:52:17,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:17,622 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/81a6943044e04c2e8a7d1b83a231b936 2024-11-18T07:52:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916397605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:17,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:17,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916397607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:17,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:17,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916397605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:17,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:17,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/4d46bb9beda24feebb09f34e1b9fff60 is 50, key is test_row_0/C:col10/1731916336034/Put/seqid=0 2024-11-18T07:52:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T07:52:17,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741875_1051 (size=12151) 2024-11-18T07:52:17,691 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/4d46bb9beda24feebb09f34e1b9fff60 2024-11-18T07:52:17,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/1ab81b0ad45b419cb53cc434b2234d30 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/1ab81b0ad45b419cb53cc434b2234d30 2024-11-18T07:52:17,763 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/1ab81b0ad45b419cb53cc434b2234d30, entries=150, sequenceid=247, filesize=11.9 K 2024-11-18T07:52:17,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/81a6943044e04c2e8a7d1b83a231b936 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/81a6943044e04c2e8a7d1b83a231b936 2024-11-18T07:52:17,779 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/81a6943044e04c2e8a7d1b83a231b936, entries=150, sequenceid=247, filesize=11.9 K 2024-11-18T07:52:17,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/4d46bb9beda24feebb09f34e1b9fff60 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/4d46bb9beda24feebb09f34e1b9fff60 2024-11-18T07:52:17,791 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/4d46bb9beda24feebb09f34e1b9fff60, entries=150, sequenceid=247, filesize=11.9 K 2024-11-18T07:52:17,792 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 1563b121b61f4db988d82be75c71ace9 in 773ms, sequenceid=247, compaction requested=false 2024-11-18T07:52:17,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:17,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:17,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-18T07:52:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-18T07:52:17,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-18T07:52:17,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2610 sec 2024-11-18T07:52:17,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.2710 sec 2024-11-18T07:52:17,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T07:52:17,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:17,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:17,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:17,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:17,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:17,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:17,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/15854bac2f344d0e9f8bd759f2640a79 is 50, key is test_row_0/A:col10/1731916337899/Put/seqid=0 2024-11-18T07:52:17,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741876_1052 (size=14591) 2024-11-18T07:52:18,025 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/e435353b60c64be29f300ab578eb0b6e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e435353b60c64be29f300ab578eb0b6e 2024-11-18T07:52:18,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916398004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916398006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916337990, max timestamp=1731916338026Num uniques 859; 2024-11-18T07:52:18,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:52:18,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916337990, max timestamp=1731916338026Num uniques 859; 2024-11-18T07:52:18,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:52:18,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916337990, max timestamp=1731916338026Num uniques 859; 2024-11-18T07:52:18,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:52:18,054 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/C of 1563b121b61f4db988d82be75c71ace9 into e435353b60c64be29f300ab578eb0b6e(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:18,054 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:18,054 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/C, priority=12, startTime=1731916336939; duration=0sec 2024-11-18T07:52:18,054 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:18,054 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:C 2024-11-18T07:52:18,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916398123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916398128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916398132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916398139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916398139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916398240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916398240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916398241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/15854bac2f344d0e9f8bd759f2640a79 2024-11-18T07:52:18,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916398348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916398348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/cf335aac179247b2bce916343d589610 is 50, key is test_row_0/B:col10/1731916337899/Put/seqid=0 2024-11-18T07:52:18,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741877_1053 (size=12201) 2024-11-18T07:52:18,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/cf335aac179247b2bce916343d589610 2024-11-18T07:52:18,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/38b5838db4204a71bfd21feacd50746b is 50, key is test_row_0/C:col10/1731916337899/Put/seqid=0 2024-11-18T07:52:18,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916398455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916398460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916398460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741878_1054 (size=12201) 2024-11-18T07:52:18,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916398660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916398660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T07:52:18,698 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:18,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:52:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-18T07:52:18,703 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:52:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T07:52:18,705 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:52:18,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:52:18,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916398769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916398778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916398779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:18,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T07:52:18,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-18T07:52:18,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:18,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:18,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:18,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:18,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:18,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/38b5838db4204a71bfd21feacd50746b 2024-11-18T07:52:18,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/15854bac2f344d0e9f8bd759f2640a79 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/15854bac2f344d0e9f8bd759f2640a79 2024-11-18T07:52:18,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/15854bac2f344d0e9f8bd759f2640a79, entries=200, sequenceid=260, filesize=14.2 K 2024-11-18T07:52:18,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/cf335aac179247b2bce916343d589610 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/cf335aac179247b2bce916343d589610 2024-11-18T07:52:18,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/cf335aac179247b2bce916343d589610, entries=150, sequenceid=260, filesize=11.9 K 2024-11-18T07:52:18,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/38b5838db4204a71bfd21feacd50746b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/38b5838db4204a71bfd21feacd50746b 2024-11-18T07:52:18,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/38b5838db4204a71bfd21feacd50746b, entries=150, sequenceid=260, filesize=11.9 K 2024-11-18T07:52:18,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1563b121b61f4db988d82be75c71ace9 in 1061ms, sequenceid=260, compaction requested=true 2024-11-18T07:52:18,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:18,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:18,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:18,962 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:18,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:18,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:18,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:18,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:52:18,963 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:18,964 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36879 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:18,964 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39269 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:18,965 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/A is initiating minor compaction (all files) 2024-11-18T07:52:18,965 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/B is initiating minor compaction (all files) 2024-11-18T07:52:18,965 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/A in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:18,965 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/B in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:18,965 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/71331fd0dde74b6d8a6d158d62c918c6, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/1ab81b0ad45b419cb53cc434b2234d30, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/15854bac2f344d0e9f8bd759f2640a79] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=38.3 K 2024-11-18T07:52:18,965 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/216fb691347a46459a1536638d3f6b5d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/81a6943044e04c2e8a7d1b83a231b936, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/cf335aac179247b2bce916343d589610] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=36.0 K 2024-11-18T07:52:18,966 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 216fb691347a46459a1536638d3f6b5d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731916335869 2024-11-18T07:52:18,966 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 71331fd0dde74b6d8a6d158d62c918c6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731916335869 2024-11-18T07:52:18,966 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ab81b0ad45b419cb53cc434b2234d30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731916336012 2024-11-18T07:52:18,966 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81a6943044e04c2e8a7d1b83a231b936, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731916336012 2024-11-18T07:52:18,967 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 15854bac2f344d0e9f8bd759f2640a79, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731916337231 2024-11-18T07:52:18,967 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf335aac179247b2bce916343d589610, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731916337231 2024-11-18T07:52:18,988 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#B#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:18,989 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/7759255140d2429489426aec8fe67426 is 50, key is test_row_0/B:col10/1731916337899/Put/seqid=0 2024-11-18T07:52:19,001 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:19,002 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/ba5656e3257e494d86009a742bfdb5eb is 50, key is test_row_0/A:col10/1731916337899/Put/seqid=0 2024-11-18T07:52:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T07:52:19,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-18T07:52:19,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:19,040 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-18T07:52:19,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:19,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:19,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:19,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:19,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:19,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:19,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741879_1055 (size=12679) 2024-11-18T07:52:19,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741880_1056 (size=12679) 2024-11-18T07:52:19,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/2809eb3d6be94a3f9b912c42745b4568 is 50, key is test_row_0/A:col10/1731916338124/Put/seqid=0 2024-11-18T07:52:19,087 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/ba5656e3257e494d86009a742bfdb5eb as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/ba5656e3257e494d86009a742bfdb5eb 2024-11-18T07:52:19,098 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/A of 1563b121b61f4db988d82be75c71ace9 into ba5656e3257e494d86009a742bfdb5eb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:19,099 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:19,099 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/A, priority=13, startTime=1731916338962; duration=0sec 2024-11-18T07:52:19,099 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:19,099 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:A 2024-11-18T07:52:19,099 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:19,102 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36879 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:19,103 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/C is initiating minor compaction (all files) 2024-11-18T07:52:19,103 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/C in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:19,103 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e435353b60c64be29f300ab578eb0b6e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/4d46bb9beda24feebb09f34e1b9fff60, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/38b5838db4204a71bfd21feacd50746b] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=36.0 K 2024-11-18T07:52:19,104 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting e435353b60c64be29f300ab578eb0b6e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731916335869 2024-11-18T07:52:19,104 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d46bb9beda24feebb09f34e1b9fff60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731916336012 2024-11-18T07:52:19,105 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 38b5838db4204a71bfd21feacd50746b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731916337231 2024-11-18T07:52:19,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741881_1057 (size=12301) 2024-11-18T07:52:19,145 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:19,146 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/ee9f6571b94c4cb4b50312983a3f0219 is 50, key is test_row_0/C:col10/1731916337899/Put/seqid=0 2024-11-18T07:52:19,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741882_1058 (size=12679) 2024-11-18T07:52:19,202 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/ee9f6571b94c4cb4b50312983a3f0219 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/ee9f6571b94c4cb4b50312983a3f0219 2024-11-18T07:52:19,219 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/C of 1563b121b61f4db988d82be75c71ace9 into ee9f6571b94c4cb4b50312983a3f0219(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:19,220 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:19,220 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/C, priority=13, startTime=1731916338962; duration=0sec 2024-11-18T07:52:19,220 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:19,220 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:C 2024-11-18T07:52:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:19,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:19,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916399259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:19,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916399265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:19,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:19,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916399288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916399288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:19,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916399298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T07:52:19,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916399368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:19,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916399378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:19,486 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/7759255140d2429489426aec8fe67426 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/7759255140d2429489426aec8fe67426 2024-11-18T07:52:19,508 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/B of 1563b121b61f4db988d82be75c71ace9 into 7759255140d2429489426aec8fe67426(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:19,508 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:19,508 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/B, priority=13, startTime=1731916338962; duration=0sec 2024-11-18T07:52:19,508 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:19,508 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:B 2024-11-18T07:52:19,535 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/2809eb3d6be94a3f9b912c42745b4568 2024-11-18T07:52:19,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/f30cd49e5e894660a206ed585ce70f99 is 50, key is test_row_0/B:col10/1731916338124/Put/seqid=0 2024-11-18T07:52:19,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916399579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:19,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916399588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:19,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741883_1059 (size=12301) 2024-11-18T07:52:19,614 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/f30cd49e5e894660a206ed585ce70f99 2024-11-18T07:52:19,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/91bb59deb6e844d08fdb021a0814d2dc is 50, key is test_row_0/C:col10/1731916338124/Put/seqid=0 2024-11-18T07:52:19,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741884_1060 (size=12301) 2024-11-18T07:52:19,692 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/91bb59deb6e844d08fdb021a0814d2dc 2024-11-18T07:52:19,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/2809eb3d6be94a3f9b912c42745b4568 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2809eb3d6be94a3f9b912c42745b4568 2024-11-18T07:52:19,719 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2809eb3d6be94a3f9b912c42745b4568, entries=150, sequenceid=296, filesize=12.0 K 2024-11-18T07:52:19,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/f30cd49e5e894660a206ed585ce70f99 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/f30cd49e5e894660a206ed585ce70f99 2024-11-18T07:52:19,741 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/f30cd49e5e894660a206ed585ce70f99, entries=150, sequenceid=296, filesize=12.0 K 2024-11-18T07:52:19,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/91bb59deb6e844d08fdb021a0814d2dc as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/91bb59deb6e844d08fdb021a0814d2dc 2024-11-18T07:52:19,762 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/91bb59deb6e844d08fdb021a0814d2dc, entries=150, sequenceid=296, filesize=12.0 K 2024-11-18T07:52:19,764 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 1563b121b61f4db988d82be75c71ace9 in 724ms, sequenceid=296, compaction requested=false 2024-11-18T07:52:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-18T07:52:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-18T07:52:19,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-18T07:52:19,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0620 sec 2024-11-18T07:52:19,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.0740 sec 2024-11-18T07:52:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T07:52:19,839 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:19,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:52:19,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-18T07:52:19,843 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:52:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T07:52:19,845 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:52:19,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:52:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:19,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T07:52:19,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:19,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:19,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:19,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:19,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:19,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:19,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/710b6cd6303a498b83ee6e0ce9e75611 is 50, key is test_row_0/A:col10/1731916339251/Put/seqid=0 2024-11-18T07:52:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T07:52:19,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741885_1061 (size=14741) 2024-11-18T07:52:20,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T07:52:20,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:20,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:20,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:20,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916340020, max timestamp=1731916340076Num uniques 859; 2024-11-18T07:52:20,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:52:20,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916340020, max timestamp=1731916340076Num uniques 859; 2024-11-18T07:52:20,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:52:20,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916340020, max timestamp=1731916340076Num uniques 859; 2024-11-18T07:52:20,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:52:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T07:52:20,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T07:52:20,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:20,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:20,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:20,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916400181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:20,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916400189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:20,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916400299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:20,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916400299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:20,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916400309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:20,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916400309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:20,314 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_1' in 1563b121b61f4db988d82be75c71ace9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:52:20,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:20,315 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_2' in 1563b121b61f4db988d82be75c71ace9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:52:20,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:20,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916400317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:20,323 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_1' in 1563b121b61f4db988d82be75c71ace9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:52:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:20,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T07:52:20,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:20,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:20,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:20,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/710b6cd6303a498b83ee6e0ce9e75611 2024-11-18T07:52:20,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/712fc2d95ae344b6b3b6c6ba7fa89e95 is 50, key is test_row_0/B:col10/1731916339251/Put/seqid=0 2024-11-18T07:52:20,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741886_1062 (size=12301) 2024-11-18T07:52:20,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/712fc2d95ae344b6b3b6c6ba7fa89e95 2024-11-18T07:52:20,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/1e86788225df4cf386e1072ce538c4c1 is 50, key is test_row_0/C:col10/1731916339251/Put/seqid=0 2024-11-18T07:52:20,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741887_1063 (size=12301) 2024-11-18T07:52:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T07:52:20,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T07:52:20,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:20,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:20,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:20,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916400508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:20,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916400508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:20,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T07:52:20,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:20,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:20,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:20,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T07:52:20,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:20,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:20,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:20,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:20,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916400818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:20,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916400818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:20,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:20,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:20,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/1e86788225df4cf386e1072ce538c4c1 2024-11-18T07:52:20,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/710b6cd6303a498b83ee6e0ce9e75611 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/710b6cd6303a498b83ee6e0ce9e75611 2024-11-18T07:52:20,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/710b6cd6303a498b83ee6e0ce9e75611, entries=200, sequenceid=311, filesize=14.4 K 2024-11-18T07:52:20,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/712fc2d95ae344b6b3b6c6ba7fa89e95 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/712fc2d95ae344b6b3b6c6ba7fa89e95 2024-11-18T07:52:20,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/712fc2d95ae344b6b3b6c6ba7fa89e95, entries=150, sequenceid=311, filesize=12.0 K 2024-11-18T07:52:20,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/1e86788225df4cf386e1072ce538c4c1 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/1e86788225df4cf386e1072ce538c4c1 2024-11-18T07:52:20,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/1e86788225df4cf386e1072ce538c4c1, entries=150, sequenceid=311, filesize=12.0 K 2024-11-18T07:52:20,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 1563b121b61f4db988d82be75c71ace9 in 1002ms, sequenceid=311, compaction requested=true 2024-11-18T07:52:20,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:20,905 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:20,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:20,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:20,906 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:20,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:20,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:20,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:20,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:20,907 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:20,907 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/A is initiating minor compaction (all files) 2024-11-18T07:52:20,907 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/A in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:20,907 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/ba5656e3257e494d86009a742bfdb5eb, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2809eb3d6be94a3f9b912c42745b4568, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/710b6cd6303a498b83ee6e0ce9e75611] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=38.8 K 2024-11-18T07:52:20,908 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37281 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:20,908 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/B is initiating minor compaction (all files) 2024-11-18T07:52:20,908 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/B in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:20,908 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/7759255140d2429489426aec8fe67426, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/f30cd49e5e894660a206ed585ce70f99, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/712fc2d95ae344b6b3b6c6ba7fa89e95] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=36.4 K 2024-11-18T07:52:20,908 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba5656e3257e494d86009a742bfdb5eb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731916337231 2024-11-18T07:52:20,909 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7759255140d2429489426aec8fe67426, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731916337231 2024-11-18T07:52:20,912 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2809eb3d6be94a3f9b912c42745b4568, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731916338100 2024-11-18T07:52:20,913 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 710b6cd6303a498b83ee6e0ce9e75611, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731916339239 2024-11-18T07:52:20,914 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting f30cd49e5e894660a206ed585ce70f99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731916338100 2024-11-18T07:52:20,914 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 712fc2d95ae344b6b3b6c6ba7fa89e95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731916339251 2024-11-18T07:52:20,935 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#A#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:20,936 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/f6f1e40cbadb490b8b7f6e3aed9d60aa is 50, key is test_row_0/A:col10/1731916339251/Put/seqid=0 2024-11-18T07:52:20,941 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:20,942 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/4af2ccafac624fdba5dbe11ee02a2190 is 50, key is test_row_0/B:col10/1731916339251/Put/seqid=0 2024-11-18T07:52:20,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T07:52:20,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:20,975 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-18T07:52:20,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:20,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:20,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:20,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:20,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:20,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T07:52:20,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741888_1064 (size=12881) 2024-11-18T07:52:21,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741889_1065 (size=12881) 2024-11-18T07:52:21,014 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/f6f1e40cbadb490b8b7f6e3aed9d60aa as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/f6f1e40cbadb490b8b7f6e3aed9d60aa 2024-11-18T07:52:21,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/8bbc5ecdd0e348ee8fbace13ad69ac56 is 50, key is test_row_0/A:col10/1731916340187/Put/seqid=0 2024-11-18T07:52:21,023 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/4af2ccafac624fdba5dbe11ee02a2190 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4af2ccafac624fdba5dbe11ee02a2190 2024-11-18T07:52:21,024 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/A of 1563b121b61f4db988d82be75c71ace9 into f6f1e40cbadb490b8b7f6e3aed9d60aa(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:21,024 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:21,024 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/A, priority=13, startTime=1731916340905; duration=0sec 2024-11-18T07:52:21,024 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:21,024 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:A 2024-11-18T07:52:21,025 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:21,028 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37281 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:21,028 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/C is initiating minor compaction (all files) 2024-11-18T07:52:21,028 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/C in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:21,028 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/ee9f6571b94c4cb4b50312983a3f0219, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/91bb59deb6e844d08fdb021a0814d2dc, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/1e86788225df4cf386e1072ce538c4c1] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=36.4 K 2024-11-18T07:52:21,029 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee9f6571b94c4cb4b50312983a3f0219, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731916337231 2024-11-18T07:52:21,030 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91bb59deb6e844d08fdb021a0814d2dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731916338100 2024-11-18T07:52:21,030 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e86788225df4cf386e1072ce538c4c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731916339251 2024-11-18T07:52:21,035 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/B of 1563b121b61f4db988d82be75c71ace9 into 4af2ccafac624fdba5dbe11ee02a2190(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:21,035 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:21,035 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/B, priority=13, startTime=1731916340906; duration=0sec 2024-11-18T07:52:21,035 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:21,035 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:B 2024-11-18T07:52:21,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741890_1066 (size=12301) 2024-11-18T07:52:21,061 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/8bbc5ecdd0e348ee8fbace13ad69ac56 2024-11-18T07:52:21,071 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#C#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:21,071 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/47a2d5314359430ba5274b16133247c3 is 50, key is test_row_0/C:col10/1731916339251/Put/seqid=0 2024-11-18T07:52:21,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/31cd07d6906740fdb325db7c9198ae51 is 50, key is test_row_0/B:col10/1731916340187/Put/seqid=0 2024-11-18T07:52:21,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741891_1067 (size=12881) 2024-11-18T07:52:21,108 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/47a2d5314359430ba5274b16133247c3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/47a2d5314359430ba5274b16133247c3 2024-11-18T07:52:21,121 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/C of 1563b121b61f4db988d82be75c71ace9 into 47a2d5314359430ba5274b16133247c3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:21,121 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:21,121 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/C, priority=13, startTime=1731916340906; duration=0sec 2024-11-18T07:52:21,123 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:21,123 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:C 2024-11-18T07:52:21,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741892_1068 (size=12301) 2024-11-18T07:52:21,150 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/31cd07d6906740fdb325db7c9198ae51 2024-11-18T07:52:21,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/98687e073c8440de91f55e2eefd110f7 is 50, key is test_row_0/C:col10/1731916340187/Put/seqid=0 2024-11-18T07:52:21,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741893_1069 (size=12301) 2024-11-18T07:52:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:21,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:21,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:21,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916401359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:21,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:21,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916401357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:21,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916401472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:21,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916401478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:21,643 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/98687e073c8440de91f55e2eefd110f7 2024-11-18T07:52:21,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/8bbc5ecdd0e348ee8fbace13ad69ac56 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/8bbc5ecdd0e348ee8fbace13ad69ac56 2024-11-18T07:52:21,678 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/8bbc5ecdd0e348ee8fbace13ad69ac56, entries=150, sequenceid=345, filesize=12.0 K 2024-11-18T07:52:21,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/31cd07d6906740fdb325db7c9198ae51 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/31cd07d6906740fdb325db7c9198ae51 2024-11-18T07:52:21,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916401688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:21,698 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/31cd07d6906740fdb325db7c9198ae51, entries=150, sequenceid=345, filesize=12.0 K 2024-11-18T07:52:21,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:21,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916401698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:21,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/98687e073c8440de91f55e2eefd110f7 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/98687e073c8440de91f55e2eefd110f7 2024-11-18T07:52:21,711 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/98687e073c8440de91f55e2eefd110f7, entries=150, sequenceid=345, filesize=12.0 K 2024-11-18T07:52:21,712 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for 1563b121b61f4db988d82be75c71ace9 in 738ms, sequenceid=345, compaction requested=false 2024-11-18T07:52:21,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:21,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:21,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-18T07:52:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-18T07:52:21,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-18T07:52:21,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8690 sec 2024-11-18T07:52:21,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.8800 sec 2024-11-18T07:52:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T07:52:21,989 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:21,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:52:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-18T07:52:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T07:52:21,993 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:52:21,993 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:52:21,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:52:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:22,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T07:52:22,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:22,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:22,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:22,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:22,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:22,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:22,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/2e11d4782a594cbf98bcb143ae281f27 is 50, key is test_row_0/A:col10/1731916342011/Put/seqid=0 2024-11-18T07:52:22,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741894_1070 (size=14741) 2024-11-18T07:52:22,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/2e11d4782a594cbf98bcb143ae281f27 2024-11-18T07:52:22,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/41af4b7c479f41f0880c0b8380bed8a9 is 50, key is test_row_0/B:col10/1731916342011/Put/seqid=0 2024-11-18T07:52:22,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916402088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T07:52:22,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916402096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741895_1071 (size=12301) 2024-11-18T07:52:22,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/41af4b7c479f41f0880c0b8380bed8a9 2024-11-18T07:52:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/f3b452743a5b4633bda07eef1f816beb is 50, key is test_row_0/C:col10/1731916342011/Put/seqid=0 2024-11-18T07:52:22,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T07:52:22,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:22,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:22,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:22,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741896_1072 (size=12301) 2024-11-18T07:52:22,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/f3b452743a5b4633bda07eef1f816beb 2024-11-18T07:52:22,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/2e11d4782a594cbf98bcb143ae281f27 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2e11d4782a594cbf98bcb143ae281f27 2024-11-18T07:52:22,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2e11d4782a594cbf98bcb143ae281f27, entries=200, sequenceid=362, filesize=14.4 K 2024-11-18T07:52:22,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/41af4b7c479f41f0880c0b8380bed8a9 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/41af4b7c479f41f0880c0b8380bed8a9 2024-11-18T07:52:22,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/41af4b7c479f41f0880c0b8380bed8a9, entries=150, sequenceid=362, filesize=12.0 K 2024-11-18T07:52:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/f3b452743a5b4633bda07eef1f816beb as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/f3b452743a5b4633bda07eef1f816beb 2024-11-18T07:52:22,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916402208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/f3b452743a5b4633bda07eef1f816beb, entries=150, sequenceid=362, filesize=12.0 K 2024-11-18T07:52:22,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1563b121b61f4db988d82be75c71ace9 in 216ms, sequenceid=362, compaction requested=true 2024-11-18T07:52:22,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:22,229 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:22,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:22,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:22,229 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:22,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:22,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:22,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:22,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:22,230 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:22,230 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/A is initiating minor compaction (all files) 2024-11-18T07:52:22,230 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/A in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:22,231 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/f6f1e40cbadb490b8b7f6e3aed9d60aa, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/8bbc5ecdd0e348ee8fbace13ad69ac56, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2e11d4782a594cbf98bcb143ae281f27] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=39.0 K 2024-11-18T07:52:22,233 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:22,233 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/B is initiating minor compaction (all files) 2024-11-18T07:52:22,233 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/B in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:22,233 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4af2ccafac624fdba5dbe11ee02a2190, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/31cd07d6906740fdb325db7c9198ae51, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/41af4b7c479f41f0880c0b8380bed8a9] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=36.6 K 2024-11-18T07:52:22,233 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6f1e40cbadb490b8b7f6e3aed9d60aa, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731916339251 2024-11-18T07:52:22,235 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4af2ccafac624fdba5dbe11ee02a2190, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731916339251 2024-11-18T07:52:22,235 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bbc5ecdd0e348ee8fbace13ad69ac56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731916340155 2024-11-18T07:52:22,237 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e11d4782a594cbf98bcb143ae281f27, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731916341338 2024-11-18T07:52:22,237 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 31cd07d6906740fdb325db7c9198ae51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731916340155 2024-11-18T07:52:22,240 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 41af4b7c479f41f0880c0b8380bed8a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731916341347 2024-11-18T07:52:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:22,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T07:52:22,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:22,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:22,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:22,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:22,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:22,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:22,275 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:22,276 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/bb56e1dd0770472d918ada8b1f25af3f is 50, key is test_row_0/B:col10/1731916342011/Put/seqid=0 2024-11-18T07:52:22,278 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#A#compaction#60 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:22,279 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/2642fd85a93841fea695d846210a13fb is 50, key is test_row_0/A:col10/1731916342011/Put/seqid=0 2024-11-18T07:52:22,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/d8d8cccce6f649728b93d10223954c92 is 50, key is test_row_0/A:col10/1731916342078/Put/seqid=0 2024-11-18T07:52:22,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T07:52:22,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:22,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:22,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:22,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T07:52:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741897_1073 (size=12983) 2024-11-18T07:52:22,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741899_1075 (size=14741) 2024-11-18T07:52:22,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/d8d8cccce6f649728b93d10223954c92 2024-11-18T07:52:22,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916402343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741898_1074 (size=12983) 2024-11-18T07:52:22,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916402349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/6bfbb688012f429cb5f171989bea5709 is 50, key is test_row_0/B:col10/1731916342078/Put/seqid=0 2024-11-18T07:52:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916402353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916402356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,383 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/bb56e1dd0770472d918ada8b1f25af3f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/bb56e1dd0770472d918ada8b1f25af3f 2024-11-18T07:52:22,391 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/B of 1563b121b61f4db988d82be75c71ace9 into bb56e1dd0770472d918ada8b1f25af3f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:22,391 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:22,391 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/B, priority=13, startTime=1731916342229; duration=0sec 2024-11-18T07:52:22,391 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:22,391 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:B 2024-11-18T07:52:22,391 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:22,398 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:22,398 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/C is initiating minor compaction (all files) 2024-11-18T07:52:22,398 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/C in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:22,398 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/47a2d5314359430ba5274b16133247c3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/98687e073c8440de91f55e2eefd110f7, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/f3b452743a5b4633bda07eef1f816beb] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=36.6 K 2024-11-18T07:52:22,402 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 47a2d5314359430ba5274b16133247c3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731916339251 2024-11-18T07:52:22,403 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 98687e073c8440de91f55e2eefd110f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731916340155 2024-11-18T07:52:22,403 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting f3b452743a5b4633bda07eef1f816beb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731916341347 2024-11-18T07:52:22,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741900_1076 (size=12301) 2024-11-18T07:52:22,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/6bfbb688012f429cb5f171989bea5709 2024-11-18T07:52:22,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916402428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/7a213b5edec24d1886019d50bce7856b is 50, key is test_row_0/C:col10/1731916342078/Put/seqid=0 2024-11-18T07:52:22,441 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#C#compaction#64 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:22,442 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/7650991ed3904cbe97b8b83af09ca21b is 50, key is test_row_0/C:col10/1731916342011/Put/seqid=0 2024-11-18T07:52:22,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T07:52:22,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:22,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:22,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:22,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916402458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916402469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916402478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916402488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741901_1077 (size=12301) 2024-11-18T07:52:22,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741902_1078 (size=12983) 2024-11-18T07:52:22,536 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/7650991ed3904cbe97b8b83af09ca21b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7650991ed3904cbe97b8b83af09ca21b 2024-11-18T07:52:22,550 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/C of 1563b121b61f4db988d82be75c71ace9 into 7650991ed3904cbe97b8b83af09ca21b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:22,550 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:22,550 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/C, priority=13, startTime=1731916342230; duration=0sec 2024-11-18T07:52:22,551 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:22,551 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:C 2024-11-18T07:52:22,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T07:52:22,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:22,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:22,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:22,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T07:52:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916402678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916402688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916402698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916402708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,739 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/2642fd85a93841fea695d846210a13fb as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2642fd85a93841fea695d846210a13fb 2024-11-18T07:52:22,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916402748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:22,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:22,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:22,757 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/A of 1563b121b61f4db988d82be75c71ace9 into 2642fd85a93841fea695d846210a13fb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:22,757 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:22,757 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/A, priority=13, startTime=1731916342229; duration=0sec 2024-11-18T07:52:22,757 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:22,757 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:A 2024-11-18T07:52:22,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T07:52:22,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:22,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:22,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:22,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/7a213b5edec24d1886019d50bce7856b 2024-11-18T07:52:22,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T07:52:22,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:22,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:22,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:22,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:22,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/d8d8cccce6f649728b93d10223954c92 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/d8d8cccce6f649728b93d10223954c92 2024-11-18T07:52:22,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/d8d8cccce6f649728b93d10223954c92, entries=200, sequenceid=382, filesize=14.4 K 2024-11-18T07:52:22,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/6bfbb688012f429cb5f171989bea5709 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6bfbb688012f429cb5f171989bea5709 2024-11-18T07:52:22,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6bfbb688012f429cb5f171989bea5709, entries=150, sequenceid=382, filesize=12.0 K 2024-11-18T07:52:22,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/7a213b5edec24d1886019d50bce7856b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7a213b5edec24d1886019d50bce7856b 2024-11-18T07:52:22,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7a213b5edec24d1886019d50bce7856b, entries=150, sequenceid=382, filesize=12.0 K 2024-11-18T07:52:22,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1563b121b61f4db988d82be75c71ace9 in 711ms, sequenceid=382, compaction requested=false 2024-11-18T07:52:22,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:22,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T07:52:22,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:22,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:22,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:22,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:22,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:22,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:23,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/ab46ef92c87646f4ae8b0f572d336cb8 is 50, key is test_row_0/A:col10/1731916342287/Put/seqid=0 2024-11-18T07:52:23,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741903_1079 (size=14741) 2024-11-18T07:52:23,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/ab46ef92c87646f4ae8b0f572d336cb8 2024-11-18T07:52:23,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/22a3507095b44aaca3e2d06038e33cac is 50, key is test_row_0/B:col10/1731916342287/Put/seqid=0 2024-11-18T07:52:23,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916403039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916403045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916403045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916403045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741904_1080 (size=12301) 2024-11-18T07:52:23,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/22a3507095b44aaca3e2d06038e33cac 2024-11-18T07:52:23,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T07:52:23,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:23,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:23,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:23,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/692a606e74d049be9b487e38a760d9d9 is 50, key is test_row_0/C:col10/1731916342287/Put/seqid=0 2024-11-18T07:52:23,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741905_1081 (size=12301) 2024-11-18T07:52:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T07:52:23,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916403148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916403158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916403159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916403159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T07:52:23,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:23,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:23,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:23,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916403269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916403359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916403368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916403368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916403368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T07:52:23,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:23,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:23,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:23,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/692a606e74d049be9b487e38a760d9d9 2024-11-18T07:52:23,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/ab46ef92c87646f4ae8b0f572d336cb8 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/ab46ef92c87646f4ae8b0f572d336cb8 2024-11-18T07:52:23,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/ab46ef92c87646f4ae8b0f572d336cb8, entries=200, sequenceid=398, filesize=14.4 K 2024-11-18T07:52:23,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/22a3507095b44aaca3e2d06038e33cac as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/22a3507095b44aaca3e2d06038e33cac 2024-11-18T07:52:23,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/22a3507095b44aaca3e2d06038e33cac, entries=150, sequenceid=398, filesize=12.0 K 2024-11-18T07:52:23,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T07:52:23,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/692a606e74d049be9b487e38a760d9d9 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/692a606e74d049be9b487e38a760d9d9 2024-11-18T07:52:23,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:23,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:23,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:23,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/692a606e74d049be9b487e38a760d9d9, entries=150, sequenceid=398, filesize=12.0 K 2024-11-18T07:52:23,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1563b121b61f4db988d82be75c71ace9 in 567ms, sequenceid=398, compaction requested=true 2024-11-18T07:52:23,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:23,561 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:23,563 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:23,563 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/A is initiating minor compaction (all files) 2024-11-18T07:52:23,563 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/A in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:23,563 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2642fd85a93841fea695d846210a13fb, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/d8d8cccce6f649728b93d10223954c92, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/ab46ef92c87646f4ae8b0f572d336cb8] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=41.5 K 2024-11-18T07:52:23,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:23,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:23,564 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2642fd85a93841fea695d846210a13fb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731916341347 2024-11-18T07:52:23,564 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:23,564 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8d8cccce6f649728b93d10223954c92, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731916342078 2024-11-18T07:52:23,565 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:23,565 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab46ef92c87646f4ae8b0f572d336cb8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731916342287 2024-11-18T07:52:23,565 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/B is initiating minor compaction (all files) 2024-11-18T07:52:23,565 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/B in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:23,565 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/bb56e1dd0770472d918ada8b1f25af3f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6bfbb688012f429cb5f171989bea5709, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/22a3507095b44aaca3e2d06038e33cac] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=36.7 K 2024-11-18T07:52:23,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:23,566 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting bb56e1dd0770472d918ada8b1f25af3f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731916341347 2024-11-18T07:52:23,567 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bfbb688012f429cb5f171989bea5709, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731916342078 2024-11-18T07:52:23,568 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 22a3507095b44aaca3e2d06038e33cac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731916342287 2024-11-18T07:52:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:23,587 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:23,588 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/cc7f24929c5643d296f2adf496783cd2 is 50, key is test_row_0/A:col10/1731916342287/Put/seqid=0 2024-11-18T07:52:23,588 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:23,589 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/5cb7d57929a64bd9a5fdbbd13ec7d285 is 50, key is test_row_0/B:col10/1731916342287/Put/seqid=0 2024-11-18T07:52:23,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741906_1082 (size=13085) 2024-11-18T07:52:23,608 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/5cb7d57929a64bd9a5fdbbd13ec7d285 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/5cb7d57929a64bd9a5fdbbd13ec7d285 2024-11-18T07:52:23,617 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/B of 1563b121b61f4db988d82be75c71ace9 into 5cb7d57929a64bd9a5fdbbd13ec7d285(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:23,617 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:23,617 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/B, priority=13, startTime=1731916343564; duration=0sec 2024-11-18T07:52:23,617 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:23,617 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:B 2024-11-18T07:52:23,618 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:23,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741907_1083 (size=13085) 2024-11-18T07:52:23,620 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:23,620 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/C is initiating minor compaction (all files) 2024-11-18T07:52:23,620 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/C in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:23,620 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7650991ed3904cbe97b8b83af09ca21b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7a213b5edec24d1886019d50bce7856b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/692a606e74d049be9b487e38a760d9d9] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=36.7 K 2024-11-18T07:52:23,621 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7650991ed3904cbe97b8b83af09ca21b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731916341347 2024-11-18T07:52:23,621 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a213b5edec24d1886019d50bce7856b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731916342078 2024-11-18T07:52:23,622 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 692a606e74d049be9b487e38a760d9d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731916342287 2024-11-18T07:52:23,652 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#C#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:23,653 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/03add7be18c64103adad65de43b25c78 is 50, key is test_row_0/C:col10/1731916342287/Put/seqid=0 2024-11-18T07:52:23,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741908_1084 (size=13085) 2024-11-18T07:52:23,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:23,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T07:52:23,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:23,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:23,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:23,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:23,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:23,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:23,696 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/03add7be18c64103adad65de43b25c78 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/03add7be18c64103adad65de43b25c78 2024-11-18T07:52:23,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/5a528fe7b8e448979f9f492edc5ffe56 is 50, key is test_row_0/A:col10/1731916343677/Put/seqid=0 2024-11-18T07:52:23,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T07:52:23,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:23,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:23,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:23,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,713 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/C of 1563b121b61f4db988d82be75c71ace9 into 03add7be18c64103adad65de43b25c78(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:23,713 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:23,713 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/C, priority=13, startTime=1731916343571; duration=0sec 2024-11-18T07:52:23,714 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:23,714 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:C 2024-11-18T07:52:23,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916403709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916403711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916403711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916403714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741909_1085 (size=12301) 2024-11-18T07:52:23,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/5a528fe7b8e448979f9f492edc5ffe56 2024-11-18T07:52:23,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/be432398ea664086b873e14e9c2eba75 is 50, key is test_row_0/B:col10/1731916343677/Put/seqid=0 2024-11-18T07:52:23,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741910_1086 (size=12301) 2024-11-18T07:52:23,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/be432398ea664086b873e14e9c2eba75 2024-11-18T07:52:23,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/27aaa5c2806f4689a8219e92cd2854e9 is 50, key is test_row_0/C:col10/1731916343677/Put/seqid=0 2024-11-18T07:52:23,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916403828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916403828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916403829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916403829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:23,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:23,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:23,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741911_1087 (size=12301) 2024-11-18T07:52:23,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T07:52:23,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:23,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:23,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:23,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:24,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T07:52:24,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:24,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:24,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:24,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:24,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:24,033 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/cc7f24929c5643d296f2adf496783cd2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/cc7f24929c5643d296f2adf496783cd2 2024-11-18T07:52:24,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916404038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916404038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916404039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916404039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,044 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/A of 1563b121b61f4db988d82be75c71ace9 into cc7f24929c5643d296f2adf496783cd2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:24,044 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:24,044 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/A, priority=13, startTime=1731916343561; duration=0sec 2024-11-18T07:52:24,044 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:24,044 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:A 2024-11-18T07:52:24,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T07:52:24,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T07:52:24,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:24,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:24,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:24,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:24,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:24,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/27aaa5c2806f4689a8219e92cd2854e9 2024-11-18T07:52:24,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/5a528fe7b8e448979f9f492edc5ffe56 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/5a528fe7b8e448979f9f492edc5ffe56 2024-11-18T07:52:24,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/5a528fe7b8e448979f9f492edc5ffe56, entries=150, sequenceid=419, filesize=12.0 K 2024-11-18T07:52:24,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/be432398ea664086b873e14e9c2eba75 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/be432398ea664086b873e14e9c2eba75 2024-11-18T07:52:24,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/be432398ea664086b873e14e9c2eba75, entries=150, sequenceid=419, filesize=12.0 K 2024-11-18T07:52:24,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/27aaa5c2806f4689a8219e92cd2854e9 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/27aaa5c2806f4689a8219e92cd2854e9 2024-11-18T07:52:24,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916404278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,282 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_1' in 1563b121b61f4db988d82be75c71ace9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:52:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/27aaa5c2806f4689a8219e92cd2854e9, entries=150, sequenceid=419, filesize=12.0 K 2024-11-18T07:52:24,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1563b121b61f4db988d82be75c71ace9 in 612ms, sequenceid=419, compaction requested=false 2024-11-18T07:52:24,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:24,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T07:52:24,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:24,320 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T07:52:24,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:24,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:24,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:24,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:24,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:24,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:24,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/693dc17d6b364ccf8527b35156e1436c is 50, key is test_row_0/A:col10/1731916343710/Put/seqid=0 2024-11-18T07:52:24,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741912_1088 (size=12301) 2024-11-18T07:52:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:24,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:24,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916404372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916404375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916404375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916404380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916404488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916404489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916404489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916404489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916404698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916404698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916404698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916404698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:24,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:24,739 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/693dc17d6b364ccf8527b35156e1436c 2024-11-18T07:52:24,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/88aee1d86c4847ecbfbbb0be170387f0 is 50, key is test_row_0/B:col10/1731916343710/Put/seqid=0 2024-11-18T07:52:24,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741913_1089 (size=12301) 2024-11-18T07:52:24,773 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/88aee1d86c4847ecbfbbb0be170387f0 2024-11-18T07:52:24,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/6a8e46d3d39643ca95da2e67e9b760dd is 50, key is test_row_0/C:col10/1731916343710/Put/seqid=0 2024-11-18T07:52:24,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741914_1090 (size=12301) 2024-11-18T07:52:25,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916405008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:25,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:25,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916405009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:25,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:25,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916405009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:25,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916405015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:25,198 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/6a8e46d3d39643ca95da2e67e9b760dd 2024-11-18T07:52:25,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/693dc17d6b364ccf8527b35156e1436c as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/693dc17d6b364ccf8527b35156e1436c 2024-11-18T07:52:25,212 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/693dc17d6b364ccf8527b35156e1436c, entries=150, sequenceid=433, filesize=12.0 K 2024-11-18T07:52:25,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/88aee1d86c4847ecbfbbb0be170387f0 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/88aee1d86c4847ecbfbbb0be170387f0 2024-11-18T07:52:25,225 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/88aee1d86c4847ecbfbbb0be170387f0, entries=150, sequenceid=433, filesize=12.0 K 2024-11-18T07:52:25,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/6a8e46d3d39643ca95da2e67e9b760dd as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/6a8e46d3d39643ca95da2e67e9b760dd 2024-11-18T07:52:25,239 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/6a8e46d3d39643ca95da2e67e9b760dd, entries=150, sequenceid=433, filesize=12.0 K 2024-11-18T07:52:25,243 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1563b121b61f4db988d82be75c71ace9 in 924ms, sequenceid=433, compaction requested=true 2024-11-18T07:52:25,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:25,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:25,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-18T07:52:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-18T07:52:25,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-18T07:52:25,251 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2540 sec 2024-11-18T07:52:25,255 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 3.2630 sec 2024-11-18T07:52:25,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:25,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-18T07:52:25,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:25,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:25,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:25,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:25,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:25,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:25,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/28194e5f00cb46a2940080519ab4c14a is 50, key is test_row_0/A:col10/1731916345519/Put/seqid=0 2024-11-18T07:52:25,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916405553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:25,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916405554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:25,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916405555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:25,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916405559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:25,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741915_1091 (size=12301) 2024-11-18T07:52:25,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916405670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916405672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916405672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:25,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916405675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:25,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916405879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:25,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916405879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:25,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:25,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:25,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916405888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:25,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:25,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916405898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:25,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:25,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/28194e5f00cb46a2940080519ab4c14a 2024-11-18T07:52:26,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/f2e0cd617d984113af6bbcba8f4f612c is 50, key is test_row_0/B:col10/1731916345519/Put/seqid=0 2024-11-18T07:52:26,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741916_1092 (size=12301) 2024-11-18T07:52:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T07:52:26,159 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:26,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:52:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-18T07:52:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T07:52:26,162 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:52:26,171 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:52:26,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:52:26,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916406195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:26,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:26,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916406199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:26,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:26,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916406199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:26,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916406208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T07:52:26,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731916406299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:26,305 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_1' in 1563b121b61f4db988d82be75c71ace9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:52:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:26,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T07:52:26,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:26,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:26,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:26,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:26,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:26,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/f2e0cd617d984113af6bbcba8f4f612c 2024-11-18T07:52:26,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/a80002ae09eb411591bdd2dfce76f3f4 is 50, key is test_row_0/C:col10/1731916345519/Put/seqid=0 2024-11-18T07:52:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T07:52:26,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T07:52:26,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:26,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:26,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:26,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:26,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:26,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741917_1093 (size=12301) 2024-11-18T07:52:26,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/a80002ae09eb411591bdd2dfce76f3f4 2024-11-18T07:52:26,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/28194e5f00cb46a2940080519ab4c14a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/28194e5f00cb46a2940080519ab4c14a 2024-11-18T07:52:26,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/28194e5f00cb46a2940080519ab4c14a, entries=150, sequenceid=455, filesize=12.0 K 2024-11-18T07:52:26,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/f2e0cd617d984113af6bbcba8f4f612c as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/f2e0cd617d984113af6bbcba8f4f612c 2024-11-18T07:52:26,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/f2e0cd617d984113af6bbcba8f4f612c, entries=150, sequenceid=455, filesize=12.0 K 2024-11-18T07:52:26,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/a80002ae09eb411591bdd2dfce76f3f4 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/a80002ae09eb411591bdd2dfce76f3f4 2024-11-18T07:52:26,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/a80002ae09eb411591bdd2dfce76f3f4, entries=150, sequenceid=455, filesize=12.0 K 2024-11-18T07:52:26,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 1563b121b61f4db988d82be75c71ace9 in 1117ms, sequenceid=455, compaction requested=true 2024-11-18T07:52:26,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:26,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:26,645 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:26,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:26,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:26,645 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:26,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:26,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:26,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:26,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T07:52:26,650 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:26,650 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/A is initiating minor compaction (all files) 2024-11-18T07:52:26,650 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/A in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:26,650 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/cc7f24929c5643d296f2adf496783cd2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/5a528fe7b8e448979f9f492edc5ffe56, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/693dc17d6b364ccf8527b35156e1436c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/28194e5f00cb46a2940080519ab4c14a] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=48.8 K 2024-11-18T07:52:26,651 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:26,651 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/B is initiating minor compaction (all files) 2024-11-18T07:52:26,651 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/B in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:26,651 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/5cb7d57929a64bd9a5fdbbd13ec7d285, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/be432398ea664086b873e14e9c2eba75, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/88aee1d86c4847ecbfbbb0be170387f0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/f2e0cd617d984113af6bbcba8f4f612c] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=48.8 K 2024-11-18T07:52:26,652 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cb7d57929a64bd9a5fdbbd13ec7d285, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731916342287 2024-11-18T07:52:26,652 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc7f24929c5643d296f2adf496783cd2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731916342287 2024-11-18T07:52:26,659 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a528fe7b8e448979f9f492edc5ffe56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731916343037 2024-11-18T07:52:26,659 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting be432398ea664086b873e14e9c2eba75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731916343037 2024-11-18T07:52:26,660 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 88aee1d86c4847ecbfbbb0be170387f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731916343706 2024-11-18T07:52:26,661 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 693dc17d6b364ccf8527b35156e1436c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731916343706 2024-11-18T07:52:26,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:26,663 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:52:26,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:26,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:26,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:26,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:26,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:26,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:26,664 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting f2e0cd617d984113af6bbcba8f4f612c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731916344372 2024-11-18T07:52:26,664 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28194e5f00cb46a2940080519ab4c14a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731916344372 2024-11-18T07:52:26,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/a05d083f6c4545799c646a180aaadbf9 is 50, key is test_row_0/A:col10/1731916345552/Put/seqid=0 2024-11-18T07:52:26,726 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#A#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:26,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:26,727 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/0e730648a96f4ca39366bac548c345c0 is 50, key is test_row_0/A:col10/1731916345519/Put/seqid=0 2024-11-18T07:52:26,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741918_1094 (size=12301) 2024-11-18T07:52:26,747 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#B#compaction#83 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:26,747 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/d51e8a1251f34869b5522e881012cc58 is 50, key is test_row_0/B:col10/1731916345519/Put/seqid=0 2024-11-18T07:52:26,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741920_1096 (size=13221) 2024-11-18T07:52:26,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741919_1095 (size=13221) 2024-11-18T07:52:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T07:52:26,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916406795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:26,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916406796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:26,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916406798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:26,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916406800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:26,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916406908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:26,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916406909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916406909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:26,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916406918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:27,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916407118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:27,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916407118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916407119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:27,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916407137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:27,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:27,152 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/a05d083f6c4545799c646a180aaadbf9 2024-11-18T07:52:27,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/530ae7fa6d304e408b362774f35b3848 is 50, key is test_row_0/B:col10/1731916345552/Put/seqid=0 2024-11-18T07:52:27,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741921_1097 (size=12301) 2024-11-18T07:52:27,184 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/530ae7fa6d304e408b362774f35b3848 2024-11-18T07:52:27,202 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/0e730648a96f4ca39366bac548c345c0 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/0e730648a96f4ca39366bac548c345c0 2024-11-18T07:52:27,207 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/d51e8a1251f34869b5522e881012cc58 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/d51e8a1251f34869b5522e881012cc58 2024-11-18T07:52:27,217 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/A of 1563b121b61f4db988d82be75c71ace9 into 0e730648a96f4ca39366bac548c345c0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:27,217 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:27,217 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/A, priority=12, startTime=1731916346645; duration=0sec 2024-11-18T07:52:27,217 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:27,217 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:A 2024-11-18T07:52:27,217 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:27,219 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:27,219 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/C is initiating minor compaction (all files) 2024-11-18T07:52:27,219 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/C in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:27,219 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/03add7be18c64103adad65de43b25c78, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/27aaa5c2806f4689a8219e92cd2854e9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/6a8e46d3d39643ca95da2e67e9b760dd, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/a80002ae09eb411591bdd2dfce76f3f4] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=48.8 K 2024-11-18T07:52:27,220 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03add7be18c64103adad65de43b25c78, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731916342287 2024-11-18T07:52:27,221 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27aaa5c2806f4689a8219e92cd2854e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731916343037 2024-11-18T07:52:27,222 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a8e46d3d39643ca95da2e67e9b760dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731916343706 2024-11-18T07:52:27,223 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a80002ae09eb411591bdd2dfce76f3f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731916344372 2024-11-18T07:52:27,225 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/B of 1563b121b61f4db988d82be75c71ace9 into d51e8a1251f34869b5522e881012cc58(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:27,225 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:27,225 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/B, priority=12, startTime=1731916346645; duration=0sec 2024-11-18T07:52:27,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/e4919db4b68742dd83b6052d7d9ce0bb is 50, key is test_row_0/C:col10/1731916345552/Put/seqid=0 2024-11-18T07:52:27,226 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:27,226 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:B 2024-11-18T07:52:27,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741922_1098 (size=12301) 2024-11-18T07:52:27,235 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/e4919db4b68742dd83b6052d7d9ce0bb 2024-11-18T07:52:27,237 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#C#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:27,237 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/23044081328f4d5aa35d9dff37400585 is 50, key is test_row_0/C:col10/1731916345519/Put/seqid=0 2024-11-18T07:52:27,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/a05d083f6c4545799c646a180aaadbf9 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a05d083f6c4545799c646a180aaadbf9 2024-11-18T07:52:27,249 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a05d083f6c4545799c646a180aaadbf9, entries=150, sequenceid=465, filesize=12.0 K 2024-11-18T07:52:27,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/530ae7fa6d304e408b362774f35b3848 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/530ae7fa6d304e408b362774f35b3848 2024-11-18T07:52:27,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741923_1099 (size=13221) 2024-11-18T07:52:27,260 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/530ae7fa6d304e408b362774f35b3848, entries=150, sequenceid=465, filesize=12.0 K 2024-11-18T07:52:27,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/e4919db4b68742dd83b6052d7d9ce0bb as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e4919db4b68742dd83b6052d7d9ce0bb 2024-11-18T07:52:27,271 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e4919db4b68742dd83b6052d7d9ce0bb, entries=150, sequenceid=465, filesize=12.0 K 2024-11-18T07:52:27,272 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for 1563b121b61f4db988d82be75c71ace9 in 609ms, sequenceid=465, compaction requested=false 2024-11-18T07:52:27,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:27,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:27,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-18T07:52:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-18T07:52:27,276 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/23044081328f4d5aa35d9dff37400585 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/23044081328f4d5aa35d9dff37400585 2024-11-18T07:52:27,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-18T07:52:27,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1040 sec 2024-11-18T07:52:27,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.1200 sec 2024-11-18T07:52:27,290 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/C of 1563b121b61f4db988d82be75c71ace9 into 23044081328f4d5aa35d9dff37400585(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:27,290 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:27,290 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/C, priority=12, startTime=1731916346645; duration=0sec 2024-11-18T07:52:27,290 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:27,290 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:C 2024-11-18T07:52:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T07:52:27,308 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:27,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:52:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-18T07:52:27,313 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:52:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T07:52:27,314 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:52:27,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:52:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T07:52:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:27,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=134.18 KB heapSize=399.89 KB 2024-11-18T07:52:27,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:27,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:27,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:27,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:27,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:27,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:27,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916407440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:27,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916407444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:27,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/315aabe73db54917befa6c5e25c552ba is 50, key is test_row_0/A:col10/1731916347432/Put/seqid=0 2024-11-18T07:52:27,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916407445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:27,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916407450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:27,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:27,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741924_1100 (size=17181) 2024-11-18T07:52:27,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-18T07:52:27,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:27,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:27,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:27,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:27,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:27,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916407548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:27,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:27,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916407558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:27,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916407558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:27,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-18T07:52:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T07:52:27,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:27,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:27,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:27,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:27,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:27,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916407758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:27,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:27,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916407769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:27,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:27,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916407770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:27,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-18T07:52:27,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:27,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:27,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:27,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:27,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:27,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/315aabe73db54917befa6c5e25c552ba 2024-11-18T07:52:27,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/55f8187605c04d7dad6ec4180fd0529a is 50, key is test_row_0/B:col10/1731916347432/Put/seqid=0 2024-11-18T07:52:27,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741925_1101 (size=12301) 2024-11-18T07:52:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T07:52:27,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-18T07:52:27,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:27,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:27,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:27,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916407958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:27,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:27,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:28,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916408068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:28,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:28,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916408078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:28,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916408091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:28,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-18T07:52:28,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:28,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:28,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:28,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:28,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:28,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-18T07:52:28,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:28,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. as already flushing 2024-11-18T07:52:28,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:28,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:28,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:28,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/55f8187605c04d7dad6ec4180fd0529a 2024-11-18T07:52:28,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/15249b2f6a1f4440b19bb7b4d6a5ccd1 is 50, key is test_row_0/C:col10/1731916347432/Put/seqid=0 2024-11-18T07:52:28,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741926_1102 (size=12301) 2024-11-18T07:52:28,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/15249b2f6a1f4440b19bb7b4d6a5ccd1 2024-11-18T07:52:28,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/315aabe73db54917befa6c5e25c552ba as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/315aabe73db54917befa6c5e25c552ba 2024-11-18T07:52:28,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/315aabe73db54917befa6c5e25c552ba, entries=250, sequenceid=493, filesize=16.8 K 2024-11-18T07:52:28,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/55f8187605c04d7dad6ec4180fd0529a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/55f8187605c04d7dad6ec4180fd0529a 2024-11-18T07:52:28,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/55f8187605c04d7dad6ec4180fd0529a, entries=150, sequenceid=493, filesize=12.0 K 2024-11-18T07:52:28,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/15249b2f6a1f4440b19bb7b4d6a5ccd1 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/15249b2f6a1f4440b19bb7b4d6a5ccd1 2024-11-18T07:52:28,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/15249b2f6a1f4440b19bb7b4d6a5ccd1, entries=150, sequenceid=493, filesize=12.0 K 2024-11-18T07:52:28,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~439.69 KB/450240, currentSize=26.84 KB/27480 for 1563b121b61f4db988d82be75c71ace9 in 939ms, sequenceid=493, compaction requested=true 2024-11-18T07:52:28,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:28,372 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:28,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:28,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:28,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:28,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:28,373 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:28,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:28,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:28,374 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:28,374 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/A is initiating minor compaction (all files) 2024-11-18T07:52:28,375 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/A in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:28,375 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/0e730648a96f4ca39366bac548c345c0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a05d083f6c4545799c646a180aaadbf9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/315aabe73db54917befa6c5e25c552ba] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=41.7 K 2024-11-18T07:52:28,375 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:28,375 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e730648a96f4ca39366bac548c345c0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731916344372 2024-11-18T07:52:28,375 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/B is initiating minor compaction (all files) 2024-11-18T07:52:28,375 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/B in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:28,376 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/d51e8a1251f34869b5522e881012cc58, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/530ae7fa6d304e408b362774f35b3848, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/55f8187605c04d7dad6ec4180fd0529a] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=36.9 K 2024-11-18T07:52:28,376 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting a05d083f6c4545799c646a180aaadbf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731916345536 2024-11-18T07:52:28,376 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d51e8a1251f34869b5522e881012cc58, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731916344372 2024-11-18T07:52:28,376 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 315aabe73db54917befa6c5e25c552ba, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731916346796 2024-11-18T07:52:28,377 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 530ae7fa6d304e408b362774f35b3848, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731916345536 2024-11-18T07:52:28,379 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55f8187605c04d7dad6ec4180fd0529a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731916347430 2024-11-18T07:52:28,399 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#B#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:28,399 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/affdcd17829d4c3bad2df06e2dd2660c is 50, key is test_row_0/B:col10/1731916347432/Put/seqid=0 2024-11-18T07:52:28,402 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:28,403 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/cc8350baa5b0455fbc03ea605f473355 is 50, key is test_row_0/A:col10/1731916347432/Put/seqid=0 2024-11-18T07:52:28,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-18T07:52:28,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:28,407 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-18T07:52:28,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:28,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:28,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741927_1103 (size=13323) 2024-11-18T07:52:28,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:28,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:28,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:28,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:28,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/528654072bdf4e2d8d6faf17e6eae167 is 50, key is test_row_1/A:col10/1731916347443/Put/seqid=0 2024-11-18T07:52:28,425 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/affdcd17829d4c3bad2df06e2dd2660c as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/affdcd17829d4c3bad2df06e2dd2660c 2024-11-18T07:52:28,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741928_1104 (size=13323) 2024-11-18T07:52:28,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741929_1105 (size=9857) 2024-11-18T07:52:28,440 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/528654072bdf4e2d8d6faf17e6eae167 2024-11-18T07:52:28,449 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/B of 1563b121b61f4db988d82be75c71ace9 into affdcd17829d4c3bad2df06e2dd2660c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:28,449 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:28,449 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/B, priority=13, startTime=1731916348373; duration=0sec 2024-11-18T07:52:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T07:52:28,449 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:28,449 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:B 2024-11-18T07:52:28,449 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:28,451 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:28,451 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/C is initiating minor compaction (all files) 2024-11-18T07:52:28,451 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/C in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:28,451 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/23044081328f4d5aa35d9dff37400585, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e4919db4b68742dd83b6052d7d9ce0bb, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/15249b2f6a1f4440b19bb7b4d6a5ccd1] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=36.9 K 2024-11-18T07:52:28,455 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23044081328f4d5aa35d9dff37400585, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731916344372 2024-11-18T07:52:28,456 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4919db4b68742dd83b6052d7d9ce0bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731916345536 2024-11-18T07:52:28,456 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15249b2f6a1f4440b19bb7b4d6a5ccd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731916347430 2024-11-18T07:52:28,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/873f9edd27bd4c239a6f983bcc8d2a02 is 50, key is test_row_1/B:col10/1731916347443/Put/seqid=0 2024-11-18T07:52:28,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741930_1106 (size=9857) 2024-11-18T07:52:28,469 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/873f9edd27bd4c239a6f983bcc8d2a02 2024-11-18T07:52:28,475 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#C#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:28,475 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/db1bf1629be7480e86c78e82303806bd is 50, key is test_row_0/C:col10/1731916347432/Put/seqid=0 2024-11-18T07:52:28,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741931_1107 (size=13323) 2024-11-18T07:52:28,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/77c6acf29a1344abac9eb7dd9404fdb8 is 50, key is test_row_1/C:col10/1731916347443/Put/seqid=0 2024-11-18T07:52:28,494 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/db1bf1629be7480e86c78e82303806bd as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/db1bf1629be7480e86c78e82303806bd 2024-11-18T07:52:28,502 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/C of 1563b121b61f4db988d82be75c71ace9 into db1bf1629be7480e86c78e82303806bd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:28,502 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:28,502 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/C, priority=13, startTime=1731916348373; duration=0sec 2024-11-18T07:52:28,503 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:28,503 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:C 2024-11-18T07:52:28,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741932_1108 (size=9857) 2024-11-18T07:52:28,522 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/77c6acf29a1344abac9eb7dd9404fdb8 2024-11-18T07:52:28,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/528654072bdf4e2d8d6faf17e6eae167 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/528654072bdf4e2d8d6faf17e6eae167 2024-11-18T07:52:28,547 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/528654072bdf4e2d8d6faf17e6eae167, entries=100, sequenceid=500, filesize=9.6 K 2024-11-18T07:52:28,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/873f9edd27bd4c239a6f983bcc8d2a02 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/873f9edd27bd4c239a6f983bcc8d2a02 2024-11-18T07:52:28,560 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/873f9edd27bd4c239a6f983bcc8d2a02, entries=100, sequenceid=500, filesize=9.6 K 2024-11-18T07:52:28,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/77c6acf29a1344abac9eb7dd9404fdb8 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/77c6acf29a1344abac9eb7dd9404fdb8 2024-11-18T07:52:28,574 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/77c6acf29a1344abac9eb7dd9404fdb8, entries=100, sequenceid=500, filesize=9.6 K 2024-11-18T07:52:28,576 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1563b121b61f4db988d82be75c71ace9 in 168ms, sequenceid=500, compaction requested=false 2024-11-18T07:52:28,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:28,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:28,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-18T07:52:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-18T07:52:28,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-18T07:52:28,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2640 sec 2024-11-18T07:52:28,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.2720 sec 2024-11-18T07:52:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:28,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:52:28,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:28,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:28,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:28,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:28,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:28,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:28,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/a6403bec7f0c49e5b3245e20b6d1c707 is 50, key is test_row_0/A:col10/1731916348594/Put/seqid=0 2024-11-18T07:52:28,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741933_1109 (size=14741) 2024-11-18T07:52:28,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731916348621, max timestamp=1731916348650Num uniques 859; 2024-11-18T07:52:28,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731916348621, max timestamp=1731916348650Num uniques 859; 2024-11-18T07:52:28,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:52:28,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:52:28,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731916348621, max timestamp=1731916348650Num uniques 859; 2024-11-18T07:52:28,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:52:28,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916408692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:28,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916408694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:28,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916408695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:28,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916408797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:28,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:28,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916408808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:28,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916408808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:28,839 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/cc8350baa5b0455fbc03ea605f473355 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/cc8350baa5b0455fbc03ea605f473355 2024-11-18T07:52:28,850 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/A of 1563b121b61f4db988d82be75c71ace9 into cc8350baa5b0455fbc03ea605f473355(size=13.0 K), total size for store is 22.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:28,850 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:28,850 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/A, priority=13, startTime=1731916348372; duration=0sec 2024-11-18T07:52:28,851 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:28,851 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:A 2024-11-18T07:52:28,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731916408968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:28,970 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_2' in 1563b121b61f4db988d82be75c71ace9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:52:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:29,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:29,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916409010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:29,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:29,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:29,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:29,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916409018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:29,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:29,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:29,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:29,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916409018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:29,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:29,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:29,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:29,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/a6403bec7f0c49e5b3245e20b6d1c707 2024-11-18T07:52:29,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/e079fad27faa4dff92c1490eb3ea2e37 is 50, key is test_row_0/B:col10/1731916348594/Put/seqid=0 2024-11-18T07:52:29,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741934_1110 (size=12301) 2024-11-18T07:52:29,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/e079fad27faa4dff92c1490eb3ea2e37 2024-11-18T07:52:29,086 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-18T07:52:29,086 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:29,090 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-18T07:52:29,091 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:29,091 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-18T07:52:29,091 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:29,093 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:29,093 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:29,093 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:29,093 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:29,093 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:29,094 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:29,094 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:29,094 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:29,094 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-18T07:52:29,094 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:29,094 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:29,095 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:29,095 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:29,095 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:29,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/1bcea92e567a4b1cbc911b84f55e4986 is 50, key is test_row_0/C:col10/1731916348594/Put/seqid=0 2024-11-18T07:52:29,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741935_1111 (size=12301) 2024-11-18T07:52:29,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54912 deadline: 1731916409318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:29,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:29,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54866 deadline: 1731916409339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54868 deadline: 1731916409339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:29,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:29,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:29,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:29,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:29,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:29,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:52:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T07:52:29,469 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:29,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/1bcea92e567a4b1cbc911b84f55e4986 2024-11-18T07:52:29,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/a6403bec7f0c49e5b3245e20b6d1c707 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a6403bec7f0c49e5b3245e20b6d1c707 2024-11-18T07:52:29,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a6403bec7f0c49e5b3245e20b6d1c707, entries=200, sequenceid=512, filesize=14.4 K 2024-11-18T07:52:29,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/e079fad27faa4dff92c1490eb3ea2e37 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/e079fad27faa4dff92c1490eb3ea2e37 2024-11-18T07:52:29,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/e079fad27faa4dff92c1490eb3ea2e37, entries=150, sequenceid=512, filesize=12.0 K 2024-11-18T07:52:29,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/1bcea92e567a4b1cbc911b84f55e4986 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/1bcea92e567a4b1cbc911b84f55e4986 2024-11-18T07:52:29,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/1bcea92e567a4b1cbc911b84f55e4986, entries=150, sequenceid=512, filesize=12.0 K 2024-11-18T07:52:29,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1563b121b61f4db988d82be75c71ace9 in 929ms, sequenceid=512, compaction requested=true 2024-11-18T07:52:29,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:29,529 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:29,529 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1563b121b61f4db988d82be75c71ace9:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:29,530 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:29,530 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37921 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:29,530 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/A is initiating minor compaction (all files) 2024-11-18T07:52:29,530 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/B is initiating minor compaction (all files) 2024-11-18T07:52:29,530 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/A in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:29,530 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/B in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:29,530 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/affdcd17829d4c3bad2df06e2dd2660c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/873f9edd27bd4c239a6f983bcc8d2a02, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/e079fad27faa4dff92c1490eb3ea2e37] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=34.6 K 2024-11-18T07:52:29,530 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/cc8350baa5b0455fbc03ea605f473355, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/528654072bdf4e2d8d6faf17e6eae167, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a6403bec7f0c49e5b3245e20b6d1c707] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=37.0 K 2024-11-18T07:52:29,530 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting affdcd17829d4c3bad2df06e2dd2660c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731916347430 2024-11-18T07:52:29,530 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc8350baa5b0455fbc03ea605f473355, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731916347430 2024-11-18T07:52:29,531 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 873f9edd27bd4c239a6f983bcc8d2a02, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731916347437 2024-11-18T07:52:29,531 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 528654072bdf4e2d8d6faf17e6eae167, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731916347437 2024-11-18T07:52:29,531 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting e079fad27faa4dff92c1490eb3ea2e37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731916348580 2024-11-18T07:52:29,531 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6403bec7f0c49e5b3245e20b6d1c707, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731916348580 2024-11-18T07:52:29,538 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#B#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:29,539 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/93a90686a5154d61aa4fb1ab592b2d97 is 50, key is test_row_0/B:col10/1731916348594/Put/seqid=0 2024-11-18T07:52:29,540 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#A#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:29,541 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/0e88e0c892af4fcfab7589d4706ff319 is 50, key is test_row_0/A:col10/1731916348594/Put/seqid=0 2024-11-18T07:52:29,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741936_1112 (size=13425) 2024-11-18T07:52:29,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741937_1113 (size=13425) 2024-11-18T07:52:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:29,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-18T07:52:29,830 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-18T07:52:29,830 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:29,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:29,830 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:29,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:29,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:29,830 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:29,830 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:29,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:29,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:29,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:29,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/b14c1944214c497fb5408bbb20c1de49 is 50, key is test_row_0/A:col10/1731916349828/Put/seqid=0 2024-11-18T07:52:29,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741938_1114 (size=12301) 2024-11-18T07:52:29,850 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-18T07:52:29,850 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-18T07:52:29,850 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:29,850 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:29,850 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:29,851 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:29,851 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:29,851 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:29,851 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:29,851 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:29,950 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/93a90686a5154d61aa4fb1ab592b2d97 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/93a90686a5154d61aa4fb1ab592b2d97 2024-11-18T07:52:29,950 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/0e88e0c892af4fcfab7589d4706ff319 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/0e88e0c892af4fcfab7589d4706ff319 2024-11-18T07:52:29,956 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/A of 1563b121b61f4db988d82be75c71ace9 into 0e88e0c892af4fcfab7589d4706ff319(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:29,956 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/B of 1563b121b61f4db988d82be75c71ace9 into 93a90686a5154d61aa4fb1ab592b2d97(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:29,956 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:29,956 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:29,956 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/A, priority=13, startTime=1731916349529; duration=0sec 2024-11-18T07:52:29,956 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/B, priority=13, startTime=1731916349529; duration=0sec 2024-11-18T07:52:29,956 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:29,957 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:29,957 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:B 2024-11-18T07:52:29,957 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:A 2024-11-18T07:52:29,957 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:29,958 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:29,958 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1563b121b61f4db988d82be75c71ace9/C is initiating minor compaction (all files) 2024-11-18T07:52:29,958 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1563b121b61f4db988d82be75c71ace9/C in TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:29,958 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/db1bf1629be7480e86c78e82303806bd, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/77c6acf29a1344abac9eb7dd9404fdb8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/1bcea92e567a4b1cbc911b84f55e4986] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp, totalSize=34.6 K 2024-11-18T07:52:29,959 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting db1bf1629be7480e86c78e82303806bd, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731916347430 2024-11-18T07:52:29,959 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 77c6acf29a1344abac9eb7dd9404fdb8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731916347437 2024-11-18T07:52:29,960 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bcea92e567a4b1cbc911b84f55e4986, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731916348580 2024-11-18T07:52:29,968 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1563b121b61f4db988d82be75c71ace9#C#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:29,968 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/1bc9a9babc114eaea8f92986a112ded3 is 50, key is test_row_0/C:col10/1731916348594/Put/seqid=0 2024-11-18T07:52:29,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741939_1115 (size=13425) 2024-11-18T07:52:30,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/b14c1944214c497fb5408bbb20c1de49 2024-11-18T07:52:30,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/e460fae429704270b6782eb060bccba7 is 50, key is test_row_0/B:col10/1731916349828/Put/seqid=0 2024-11-18T07:52:30,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741940_1116 (size=12301) 2024-11-18T07:52:30,335 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-18T07:52:30,335 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:30,335 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:30,335 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:30,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:30,379 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/1bc9a9babc114eaea8f92986a112ded3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/1bc9a9babc114eaea8f92986a112ded3 2024-11-18T07:52:30,385 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1563b121b61f4db988d82be75c71ace9/C of 1563b121b61f4db988d82be75c71ace9 into 1bc9a9babc114eaea8f92986a112ded3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:30,386 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:30,386 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., storeName=1563b121b61f4db988d82be75c71ace9/C, priority=13, startTime=1731916349529; duration=0sec 2024-11-18T07:52:30,386 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:30,386 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1563b121b61f4db988d82be75c71ace9:C 2024-11-18T07:52:30,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/e460fae429704270b6782eb060bccba7 2024-11-18T07:52:30,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/99f61d9257d94a5db3f493f1f6cad1ad is 50, key is test_row_0/C:col10/1731916349828/Put/seqid=0 2024-11-18T07:52:30,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741941_1117 (size=12301) 2024-11-18T07:52:30,990 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-18T07:52:30,990 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:30,990 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:30,990 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:30,990 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:30,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T07:52:30,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-18T07:52:30,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-18T07:52:30,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-18T07:52:30,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-18T07:52:30,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-18T07:52:30,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T07:52:30,991 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1729 2024-11-18T07:52:30,991 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1684 2024-11-18T07:52:30,991 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T07:52:30,991 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 822 2024-11-18T07:52:30,991 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2465 rows 2024-11-18T07:52:30,991 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 820 2024-11-18T07:52:30,991 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2450 rows 2024-11-18T07:52:30,991 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T07:52:30,991 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T07:52:30,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:30,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:30,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:30,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T07:52:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T07:52:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T07:52:31,002 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916351001"}]},"ts":"1731916351001"} 2024-11-18T07:52:31,004 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T07:52:31,004 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T07:52:31,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T07:52:31,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1563b121b61f4db988d82be75c71ace9, UNASSIGN}] 2024-11-18T07:52:31,011 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1563b121b61f4db988d82be75c71ace9, UNASSIGN 2024-11-18T07:52:31,013 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=1563b121b61f4db988d82be75c71ace9, regionState=CLOSING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:52:31,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1563b121b61f4db988d82be75c71ace9, UNASSIGN because future has completed 2024-11-18T07:52:31,016 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T07:52:31,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:52:31,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/99f61d9257d94a5db3f493f1f6cad1ad 2024-11-18T07:52:31,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/b14c1944214c497fb5408bbb20c1de49 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/b14c1944214c497fb5408bbb20c1de49 2024-11-18T07:52:31,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/b14c1944214c497fb5408bbb20c1de49, entries=150, sequenceid=549, filesize=12.0 K 2024-11-18T07:52:31,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/e460fae429704270b6782eb060bccba7 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/e460fae429704270b6782eb060bccba7 2024-11-18T07:52:31,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/e460fae429704270b6782eb060bccba7, entries=150, sequenceid=549, filesize=12.0 K 2024-11-18T07:52:31,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/99f61d9257d94a5db3f493f1f6cad1ad as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/99f61d9257d94a5db3f493f1f6cad1ad 2024-11-18T07:52:31,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/99f61d9257d94a5db3f493f1f6cad1ad, entries=150, sequenceid=549, filesize=12.0 K 2024-11-18T07:52:31,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=26.84 KB/27480 for 1563b121b61f4db988d82be75c71ace9 in 1260ms, sequenceid=549, compaction requested=false 2024-11-18T07:52:31,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1563b121b61f4db988d82be75c71ace9: 2024-11-18T07:52:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T07:52:31,174 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:31,174 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T07:52:31,175 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 1563b121b61f4db988d82be75c71ace9, disabling compactions & flushes 2024-11-18T07:52:31,175 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:31,175 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:31,175 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. after waiting 0 ms 2024-11-18T07:52:31,175 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:31,175 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 1563b121b61f4db988d82be75c71ace9 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-18T07:52:31,175 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=A 2024-11-18T07:52:31,175 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:31,175 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=B 2024-11-18T07:52:31,175 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:31,176 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1563b121b61f4db988d82be75c71ace9, store=C 2024-11-18T07:52:31,176 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:31,180 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/2c4cfc49ea38454a977b39adab3ec0ff is 50, key is test_row_1/A:col10/1731916350318/Put/seqid=0 2024-11-18T07:52:31,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741942_1118 (size=9857) 2024-11-18T07:52:31,235 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T07:52:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T07:52:31,585 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/2c4cfc49ea38454a977b39adab3ec0ff 2024-11-18T07:52:31,593 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/c52eb9c714bf4466979b6bc574a636a6 is 50, key is test_row_1/B:col10/1731916350318/Put/seqid=0 2024-11-18T07:52:31,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741943_1119 (size=9857) 2024-11-18T07:52:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T07:52:32,001 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/c52eb9c714bf4466979b6bc574a636a6 2024-11-18T07:52:32,008 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/9a83d70f054a4f419d7c33b4cd9845bd is 50, key is test_row_1/C:col10/1731916350318/Put/seqid=0 2024-11-18T07:52:32,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741944_1120 (size=9857) 2024-11-18T07:52:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T07:52:32,413 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/9a83d70f054a4f419d7c33b4cd9845bd 2024-11-18T07:52:32,419 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/A/2c4cfc49ea38454a977b39adab3ec0ff as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2c4cfc49ea38454a977b39adab3ec0ff 2024-11-18T07:52:32,426 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2c4cfc49ea38454a977b39adab3ec0ff, entries=100, sequenceid=559, filesize=9.6 K 2024-11-18T07:52:32,427 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/B/c52eb9c714bf4466979b6bc574a636a6 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/c52eb9c714bf4466979b6bc574a636a6 2024-11-18T07:52:32,433 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/c52eb9c714bf4466979b6bc574a636a6, entries=100, sequenceid=559, filesize=9.6 K 2024-11-18T07:52:32,434 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/.tmp/C/9a83d70f054a4f419d7c33b4cd9845bd as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/9a83d70f054a4f419d7c33b4cd9845bd 2024-11-18T07:52:32,439 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/9a83d70f054a4f419d7c33b4cd9845bd, entries=100, sequenceid=559, filesize=9.6 K 2024-11-18T07:52:32,440 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1563b121b61f4db988d82be75c71ace9 in 1265ms, sequenceid=559, compaction requested=true 2024-11-18T07:52:32,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/d8fdf21ee5e34c0eb4ce8f756570afbf, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/f6802fbd7efe4b8e8e3539ce81b1d30f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a72c372715124beb943d14fb7a843e8c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/71c5c4282f894a96b37897dd340a3428, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/5945cc1f25a24d059a00009476b0d1c1, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/afd33fc1422d468d9502672e80e456e5, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/91a9839e0e524d088b8ca73905d960c0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/976241ab61854183b1cc17bb55af06ee, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/700b669574cd49519abbf5ae652898de, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/4d95ffe3102049a4801a282b0396fba3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/71331fd0dde74b6d8a6d158d62c918c6, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/81106e9692ef4471bf70adb683c2da59, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/1ab81b0ad45b419cb53cc434b2234d30, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/15854bac2f344d0e9f8bd759f2640a79, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/ba5656e3257e494d86009a742bfdb5eb, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2809eb3d6be94a3f9b912c42745b4568, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/710b6cd6303a498b83ee6e0ce9e75611, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/f6f1e40cbadb490b8b7f6e3aed9d60aa, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/8bbc5ecdd0e348ee8fbace13ad69ac56, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2e11d4782a594cbf98bcb143ae281f27, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2642fd85a93841fea695d846210a13fb, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/d8d8cccce6f649728b93d10223954c92, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/ab46ef92c87646f4ae8b0f572d336cb8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/cc7f24929c5643d296f2adf496783cd2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/5a528fe7b8e448979f9f492edc5ffe56, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/693dc17d6b364ccf8527b35156e1436c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/0e730648a96f4ca39366bac548c345c0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/28194e5f00cb46a2940080519ab4c14a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a05d083f6c4545799c646a180aaadbf9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/315aabe73db54917befa6c5e25c552ba, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/cc8350baa5b0455fbc03ea605f473355, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/528654072bdf4e2d8d6faf17e6eae167, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a6403bec7f0c49e5b3245e20b6d1c707] to archive 2024-11-18T07:52:32,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:52:32,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/d8fdf21ee5e34c0eb4ce8f756570afbf to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/d8fdf21ee5e34c0eb4ce8f756570afbf 2024-11-18T07:52:32,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/f6802fbd7efe4b8e8e3539ce81b1d30f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/f6802fbd7efe4b8e8e3539ce81b1d30f 2024-11-18T07:52:32,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a72c372715124beb943d14fb7a843e8c to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a72c372715124beb943d14fb7a843e8c 2024-11-18T07:52:32,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/71c5c4282f894a96b37897dd340a3428 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/71c5c4282f894a96b37897dd340a3428 2024-11-18T07:52:32,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/5945cc1f25a24d059a00009476b0d1c1 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/5945cc1f25a24d059a00009476b0d1c1 2024-11-18T07:52:32,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/afd33fc1422d468d9502672e80e456e5 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/afd33fc1422d468d9502672e80e456e5 2024-11-18T07:52:32,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/91a9839e0e524d088b8ca73905d960c0 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/91a9839e0e524d088b8ca73905d960c0 2024-11-18T07:52:32,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/976241ab61854183b1cc17bb55af06ee to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/976241ab61854183b1cc17bb55af06ee 2024-11-18T07:52:32,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/700b669574cd49519abbf5ae652898de to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/700b669574cd49519abbf5ae652898de 2024-11-18T07:52:32,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/4d95ffe3102049a4801a282b0396fba3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/4d95ffe3102049a4801a282b0396fba3 2024-11-18T07:52:32,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/71331fd0dde74b6d8a6d158d62c918c6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/71331fd0dde74b6d8a6d158d62c918c6 2024-11-18T07:52:32,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/81106e9692ef4471bf70adb683c2da59 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/81106e9692ef4471bf70adb683c2da59 2024-11-18T07:52:32,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/1ab81b0ad45b419cb53cc434b2234d30 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/1ab81b0ad45b419cb53cc434b2234d30 2024-11-18T07:52:32,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/15854bac2f344d0e9f8bd759f2640a79 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/15854bac2f344d0e9f8bd759f2640a79 2024-11-18T07:52:32,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/ba5656e3257e494d86009a742bfdb5eb to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/ba5656e3257e494d86009a742bfdb5eb 2024-11-18T07:52:32,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2809eb3d6be94a3f9b912c42745b4568 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2809eb3d6be94a3f9b912c42745b4568 2024-11-18T07:52:32,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/710b6cd6303a498b83ee6e0ce9e75611 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/710b6cd6303a498b83ee6e0ce9e75611 2024-11-18T07:52:32,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/f6f1e40cbadb490b8b7f6e3aed9d60aa to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/f6f1e40cbadb490b8b7f6e3aed9d60aa 2024-11-18T07:52:32,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/8bbc5ecdd0e348ee8fbace13ad69ac56 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/8bbc5ecdd0e348ee8fbace13ad69ac56 2024-11-18T07:52:32,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2e11d4782a594cbf98bcb143ae281f27 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2e11d4782a594cbf98bcb143ae281f27 2024-11-18T07:52:32,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2642fd85a93841fea695d846210a13fb to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2642fd85a93841fea695d846210a13fb 2024-11-18T07:52:32,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/d8d8cccce6f649728b93d10223954c92 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/d8d8cccce6f649728b93d10223954c92 2024-11-18T07:52:32,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/ab46ef92c87646f4ae8b0f572d336cb8 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/ab46ef92c87646f4ae8b0f572d336cb8 2024-11-18T07:52:32,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/cc7f24929c5643d296f2adf496783cd2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/cc7f24929c5643d296f2adf496783cd2 2024-11-18T07:52:32,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/5a528fe7b8e448979f9f492edc5ffe56 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/5a528fe7b8e448979f9f492edc5ffe56 2024-11-18T07:52:32,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/693dc17d6b364ccf8527b35156e1436c to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/693dc17d6b364ccf8527b35156e1436c 2024-11-18T07:52:32,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/0e730648a96f4ca39366bac548c345c0 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/0e730648a96f4ca39366bac548c345c0 2024-11-18T07:52:32,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/28194e5f00cb46a2940080519ab4c14a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/28194e5f00cb46a2940080519ab4c14a 2024-11-18T07:52:32,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a05d083f6c4545799c646a180aaadbf9 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a05d083f6c4545799c646a180aaadbf9 2024-11-18T07:52:32,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/315aabe73db54917befa6c5e25c552ba to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/315aabe73db54917befa6c5e25c552ba 2024-11-18T07:52:32,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/cc8350baa5b0455fbc03ea605f473355 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/cc8350baa5b0455fbc03ea605f473355 2024-11-18T07:52:32,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/528654072bdf4e2d8d6faf17e6eae167 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/528654072bdf4e2d8d6faf17e6eae167 2024-11-18T07:52:32,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a6403bec7f0c49e5b3245e20b6d1c707 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/a6403bec7f0c49e5b3245e20b6d1c707 2024-11-18T07:52:32,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4fa7023dc491425fbe768a705799c475, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/7868fdc0fb6a4bf69a0cb47273a02a6a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/79d13da74e824c8db7ebc0a76ce18981, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/d2ca4aa8c8c04238a7774551ae0cfc72, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/288c55225a5949a19e99516615fc3df3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4784cc50c6304526a31fa41c3bc15099, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/38b86b77727e480a9fc743e40be7ed50, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/acdf303b952b455d9478fea68097782a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/5650997ce3e84e71abfbdfb83aaf71cc, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6322c1c6184142df9bfd05dad2f54a87, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/216fb691347a46459a1536638d3f6b5d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6e3ba50558764ca8850dbf10b0e020e0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/81a6943044e04c2e8a7d1b83a231b936, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/7759255140d2429489426aec8fe67426, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/cf335aac179247b2bce916343d589610, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/f30cd49e5e894660a206ed585ce70f99, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4af2ccafac624fdba5dbe11ee02a2190, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/712fc2d95ae344b6b3b6c6ba7fa89e95, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/31cd07d6906740fdb325db7c9198ae51, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/bb56e1dd0770472d918ada8b1f25af3f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/41af4b7c479f41f0880c0b8380bed8a9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6bfbb688012f429cb5f171989bea5709, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/5cb7d57929a64bd9a5fdbbd13ec7d285, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/22a3507095b44aaca3e2d06038e33cac, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/be432398ea664086b873e14e9c2eba75, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/88aee1d86c4847ecbfbbb0be170387f0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/d51e8a1251f34869b5522e881012cc58, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/f2e0cd617d984113af6bbcba8f4f612c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/530ae7fa6d304e408b362774f35b3848, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/affdcd17829d4c3bad2df06e2dd2660c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/55f8187605c04d7dad6ec4180fd0529a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/873f9edd27bd4c239a6f983bcc8d2a02, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/e079fad27faa4dff92c1490eb3ea2e37] to archive 2024-11-18T07:52:32,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:52:32,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4fa7023dc491425fbe768a705799c475 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4fa7023dc491425fbe768a705799c475 2024-11-18T07:52:32,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/7868fdc0fb6a4bf69a0cb47273a02a6a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/7868fdc0fb6a4bf69a0cb47273a02a6a 2024-11-18T07:52:32,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/79d13da74e824c8db7ebc0a76ce18981 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/79d13da74e824c8db7ebc0a76ce18981 2024-11-18T07:52:32,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/d2ca4aa8c8c04238a7774551ae0cfc72 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/d2ca4aa8c8c04238a7774551ae0cfc72 2024-11-18T07:52:32,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/288c55225a5949a19e99516615fc3df3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/288c55225a5949a19e99516615fc3df3 2024-11-18T07:52:32,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4784cc50c6304526a31fa41c3bc15099 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4784cc50c6304526a31fa41c3bc15099 2024-11-18T07:52:32,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/38b86b77727e480a9fc743e40be7ed50 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/38b86b77727e480a9fc743e40be7ed50 2024-11-18T07:52:32,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/acdf303b952b455d9478fea68097782a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/acdf303b952b455d9478fea68097782a 2024-11-18T07:52:32,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/5650997ce3e84e71abfbdfb83aaf71cc to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/5650997ce3e84e71abfbdfb83aaf71cc 2024-11-18T07:52:32,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6322c1c6184142df9bfd05dad2f54a87 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6322c1c6184142df9bfd05dad2f54a87 2024-11-18T07:52:32,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/216fb691347a46459a1536638d3f6b5d to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/216fb691347a46459a1536638d3f6b5d 2024-11-18T07:52:32,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6e3ba50558764ca8850dbf10b0e020e0 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6e3ba50558764ca8850dbf10b0e020e0 2024-11-18T07:52:32,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/81a6943044e04c2e8a7d1b83a231b936 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/81a6943044e04c2e8a7d1b83a231b936 2024-11-18T07:52:32,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/7759255140d2429489426aec8fe67426 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/7759255140d2429489426aec8fe67426 2024-11-18T07:52:32,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/cf335aac179247b2bce916343d589610 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/cf335aac179247b2bce916343d589610 2024-11-18T07:52:32,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/f30cd49e5e894660a206ed585ce70f99 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/f30cd49e5e894660a206ed585ce70f99 2024-11-18T07:52:32,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4af2ccafac624fdba5dbe11ee02a2190 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/4af2ccafac624fdba5dbe11ee02a2190 2024-11-18T07:52:32,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/712fc2d95ae344b6b3b6c6ba7fa89e95 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/712fc2d95ae344b6b3b6c6ba7fa89e95 2024-11-18T07:52:32,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/31cd07d6906740fdb325db7c9198ae51 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/31cd07d6906740fdb325db7c9198ae51 2024-11-18T07:52:32,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/bb56e1dd0770472d918ada8b1f25af3f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/bb56e1dd0770472d918ada8b1f25af3f 2024-11-18T07:52:32,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/41af4b7c479f41f0880c0b8380bed8a9 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/41af4b7c479f41f0880c0b8380bed8a9 2024-11-18T07:52:32,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6bfbb688012f429cb5f171989bea5709 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/6bfbb688012f429cb5f171989bea5709 2024-11-18T07:52:32,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/5cb7d57929a64bd9a5fdbbd13ec7d285 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/5cb7d57929a64bd9a5fdbbd13ec7d285 2024-11-18T07:52:32,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/22a3507095b44aaca3e2d06038e33cac to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/22a3507095b44aaca3e2d06038e33cac 2024-11-18T07:52:32,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/be432398ea664086b873e14e9c2eba75 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/be432398ea664086b873e14e9c2eba75 2024-11-18T07:52:32,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/88aee1d86c4847ecbfbbb0be170387f0 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/88aee1d86c4847ecbfbbb0be170387f0 2024-11-18T07:52:32,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/d51e8a1251f34869b5522e881012cc58 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/d51e8a1251f34869b5522e881012cc58 2024-11-18T07:52:32,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/f2e0cd617d984113af6bbcba8f4f612c to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/f2e0cd617d984113af6bbcba8f4f612c 2024-11-18T07:52:32,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/530ae7fa6d304e408b362774f35b3848 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/530ae7fa6d304e408b362774f35b3848 2024-11-18T07:52:32,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/affdcd17829d4c3bad2df06e2dd2660c to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/affdcd17829d4c3bad2df06e2dd2660c 2024-11-18T07:52:32,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/55f8187605c04d7dad6ec4180fd0529a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/55f8187605c04d7dad6ec4180fd0529a 2024-11-18T07:52:32,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/873f9edd27bd4c239a6f983bcc8d2a02 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/873f9edd27bd4c239a6f983bcc8d2a02 2024-11-18T07:52:32,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/e079fad27faa4dff92c1490eb3ea2e37 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/e079fad27faa4dff92c1490eb3ea2e37 2024-11-18T07:52:32,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e472e44ddac34fb8941ed470ae4e0549, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/0525911b865e4e3cbbb0fb5ab38f1d1e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/74750b4f07c342cfafdc07cb3d20ead7, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/05f6d0c059da4cb8afc73f60c2d78498, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/f9539de564ce46fe852c2a871d245138, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/d869bf3590b74cef9c8ef0c427ab41ce, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/c5342828cbee49d89e72d21f234c341d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/2de040b01684451e8fce6a433efa0b76, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/dcfffdc3b5ea4211adc63ee07340d1de, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/0aa4a2bfa705488dae1772ad253581aa, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e435353b60c64be29f300ab578eb0b6e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7a1fae63db84444b8c107b011aa10a68, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/4d46bb9beda24feebb09f34e1b9fff60, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/ee9f6571b94c4cb4b50312983a3f0219, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/38b5838db4204a71bfd21feacd50746b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/91bb59deb6e844d08fdb021a0814d2dc, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/47a2d5314359430ba5274b16133247c3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/1e86788225df4cf386e1072ce538c4c1, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/98687e073c8440de91f55e2eefd110f7, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7650991ed3904cbe97b8b83af09ca21b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/f3b452743a5b4633bda07eef1f816beb, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7a213b5edec24d1886019d50bce7856b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/03add7be18c64103adad65de43b25c78, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/692a606e74d049be9b487e38a760d9d9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/27aaa5c2806f4689a8219e92cd2854e9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/6a8e46d3d39643ca95da2e67e9b760dd, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/23044081328f4d5aa35d9dff37400585, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/a80002ae09eb411591bdd2dfce76f3f4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e4919db4b68742dd83b6052d7d9ce0bb, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/db1bf1629be7480e86c78e82303806bd, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/15249b2f6a1f4440b19bb7b4d6a5ccd1, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/77c6acf29a1344abac9eb7dd9404fdb8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/1bcea92e567a4b1cbc911b84f55e4986] to archive 2024-11-18T07:52:32,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:52:32,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e472e44ddac34fb8941ed470ae4e0549 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e472e44ddac34fb8941ed470ae4e0549 2024-11-18T07:52:32,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/0525911b865e4e3cbbb0fb5ab38f1d1e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/0525911b865e4e3cbbb0fb5ab38f1d1e 2024-11-18T07:52:32,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/74750b4f07c342cfafdc07cb3d20ead7 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/74750b4f07c342cfafdc07cb3d20ead7 2024-11-18T07:52:32,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/05f6d0c059da4cb8afc73f60c2d78498 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/05f6d0c059da4cb8afc73f60c2d78498 2024-11-18T07:52:32,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/f9539de564ce46fe852c2a871d245138 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/f9539de564ce46fe852c2a871d245138 2024-11-18T07:52:32,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/d869bf3590b74cef9c8ef0c427ab41ce to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/d869bf3590b74cef9c8ef0c427ab41ce 2024-11-18T07:52:32,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/c5342828cbee49d89e72d21f234c341d to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/c5342828cbee49d89e72d21f234c341d 2024-11-18T07:52:32,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/2de040b01684451e8fce6a433efa0b76 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/2de040b01684451e8fce6a433efa0b76 2024-11-18T07:52:32,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/dcfffdc3b5ea4211adc63ee07340d1de to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/dcfffdc3b5ea4211adc63ee07340d1de 2024-11-18T07:52:32,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/0aa4a2bfa705488dae1772ad253581aa to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/0aa4a2bfa705488dae1772ad253581aa 2024-11-18T07:52:32,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e435353b60c64be29f300ab578eb0b6e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e435353b60c64be29f300ab578eb0b6e 2024-11-18T07:52:32,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7a1fae63db84444b8c107b011aa10a68 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7a1fae63db84444b8c107b011aa10a68 2024-11-18T07:52:32,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/4d46bb9beda24feebb09f34e1b9fff60 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/4d46bb9beda24feebb09f34e1b9fff60 2024-11-18T07:52:32,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/ee9f6571b94c4cb4b50312983a3f0219 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/ee9f6571b94c4cb4b50312983a3f0219 2024-11-18T07:52:32,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/38b5838db4204a71bfd21feacd50746b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/38b5838db4204a71bfd21feacd50746b 2024-11-18T07:52:32,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/91bb59deb6e844d08fdb021a0814d2dc to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/91bb59deb6e844d08fdb021a0814d2dc 2024-11-18T07:52:32,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/47a2d5314359430ba5274b16133247c3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/47a2d5314359430ba5274b16133247c3 2024-11-18T07:52:32,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/1e86788225df4cf386e1072ce538c4c1 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/1e86788225df4cf386e1072ce538c4c1 2024-11-18T07:52:32,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/98687e073c8440de91f55e2eefd110f7 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/98687e073c8440de91f55e2eefd110f7 2024-11-18T07:52:32,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7650991ed3904cbe97b8b83af09ca21b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7650991ed3904cbe97b8b83af09ca21b 2024-11-18T07:52:32,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/f3b452743a5b4633bda07eef1f816beb to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/f3b452743a5b4633bda07eef1f816beb 2024-11-18T07:52:32,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7a213b5edec24d1886019d50bce7856b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/7a213b5edec24d1886019d50bce7856b 2024-11-18T07:52:32,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/03add7be18c64103adad65de43b25c78 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/03add7be18c64103adad65de43b25c78 2024-11-18T07:52:32,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/692a606e74d049be9b487e38a760d9d9 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/692a606e74d049be9b487e38a760d9d9 2024-11-18T07:52:32,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/27aaa5c2806f4689a8219e92cd2854e9 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/27aaa5c2806f4689a8219e92cd2854e9 2024-11-18T07:52:32,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/6a8e46d3d39643ca95da2e67e9b760dd to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/6a8e46d3d39643ca95da2e67e9b760dd 2024-11-18T07:52:32,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/23044081328f4d5aa35d9dff37400585 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/23044081328f4d5aa35d9dff37400585 2024-11-18T07:52:32,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/a80002ae09eb411591bdd2dfce76f3f4 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/a80002ae09eb411591bdd2dfce76f3f4 2024-11-18T07:52:32,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e4919db4b68742dd83b6052d7d9ce0bb to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/e4919db4b68742dd83b6052d7d9ce0bb 2024-11-18T07:52:32,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/db1bf1629be7480e86c78e82303806bd to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/db1bf1629be7480e86c78e82303806bd 2024-11-18T07:52:32,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/15249b2f6a1f4440b19bb7b4d6a5ccd1 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/15249b2f6a1f4440b19bb7b4d6a5ccd1 2024-11-18T07:52:32,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/77c6acf29a1344abac9eb7dd9404fdb8 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/77c6acf29a1344abac9eb7dd9404fdb8 2024-11-18T07:52:32,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/1bcea92e567a4b1cbc911b84f55e4986 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/1bcea92e567a4b1cbc911b84f55e4986 2024-11-18T07:52:32,663 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/recovered.edits/562.seqid, newMaxSeqId=562, maxSeqId=1 2024-11-18T07:52:32,664 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. 2024-11-18T07:52:32,664 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 1563b121b61f4db988d82be75c71ace9: Waiting for close lock at 1731916351175Running coprocessor pre-close hooks at 1731916351175Disabling compacts and flushes for region at 1731916351175Disabling writes for close at 1731916351175Obtaining lock to block concurrent updates at 1731916351175Preparing flush snapshotting stores in 1563b121b61f4db988d82be75c71ace9 at 1731916351175Finished memstore snapshotting TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731916351176 (+1 ms)Flushing stores of TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9. at 1731916351176Flushing 1563b121b61f4db988d82be75c71ace9/A: creating writer at 1731916351176Flushing 1563b121b61f4db988d82be75c71ace9/A: appending metadata at 1731916351179 (+3 ms)Flushing 1563b121b61f4db988d82be75c71ace9/A: closing flushed file at 1731916351179Flushing 1563b121b61f4db988d82be75c71ace9/B: creating writer at 1731916351590 (+411 ms)Flushing 1563b121b61f4db988d82be75c71ace9/B: appending metadata at 1731916351593 (+3 ms)Flushing 1563b121b61f4db988d82be75c71ace9/B: closing flushed file at 1731916351593Flushing 1563b121b61f4db988d82be75c71ace9/C: creating writer at 1731916352005 (+412 ms)Flushing 1563b121b61f4db988d82be75c71ace9/C: appending metadata at 1731916352008 (+3 ms)Flushing 1563b121b61f4db988d82be75c71ace9/C: closing flushed file at 1731916352008Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6965b6a4: reopening flushed file at 1731916352418 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c2e5069: reopening flushed file at 1731916352426 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e127fe1: reopening flushed file at 1731916352433 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1563b121b61f4db988d82be75c71ace9 in 1265ms, sequenceid=559, compaction requested=true at 1731916352440 (+7 ms)Writing region close event to WAL at 1731916352659 (+219 ms)Running coprocessor post-close hooks at 1731916352663 (+4 ms)Closed at 1731916352664 (+1 ms) 2024-11-18T07:52:32,667 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:32,667 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=1563b121b61f4db988d82be75c71ace9, regionState=CLOSED 2024-11-18T07:52:32,669 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:52:32,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-18T07:52:32,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 1563b121b61f4db988d82be75c71ace9, server=87fb3d252950,37987,1731916323128 in 1.6530 sec 2024-11-18T07:52:32,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-18T07:52:32,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1563b121b61f4db988d82be75c71ace9, UNASSIGN in 1.6620 sec 2024-11-18T07:52:32,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-18T07:52:32,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6700 sec 2024-11-18T07:52:32,679 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916352679"}]},"ts":"1731916352679"} 2024-11-18T07:52:32,681 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T07:52:32,681 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T07:52:32,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6860 sec 2024-11-18T07:52:32,725 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T07:52:33,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T07:52:33,148 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:33,152 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T07:52:33,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:52:33,156 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:52:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-18T07:52:33,158 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:52:33,163 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:33,167 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/recovered.edits] 2024-11-18T07:52:33,170 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/0e88e0c892af4fcfab7589d4706ff319 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/0e88e0c892af4fcfab7589d4706ff319 2024-11-18T07:52:33,171 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2c4cfc49ea38454a977b39adab3ec0ff to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/2c4cfc49ea38454a977b39adab3ec0ff 2024-11-18T07:52:33,172 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/b14c1944214c497fb5408bbb20c1de49 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/A/b14c1944214c497fb5408bbb20c1de49 2024-11-18T07:52:33,174 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/93a90686a5154d61aa4fb1ab592b2d97 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/93a90686a5154d61aa4fb1ab592b2d97 2024-11-18T07:52:33,175 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/c52eb9c714bf4466979b6bc574a636a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/c52eb9c714bf4466979b6bc574a636a6 2024-11-18T07:52:33,176 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/e460fae429704270b6782eb060bccba7 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/B/e460fae429704270b6782eb060bccba7 2024-11-18T07:52:33,178 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/1bc9a9babc114eaea8f92986a112ded3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/1bc9a9babc114eaea8f92986a112ded3 2024-11-18T07:52:33,179 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/99f61d9257d94a5db3f493f1f6cad1ad to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/99f61d9257d94a5db3f493f1f6cad1ad 2024-11-18T07:52:33,181 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/9a83d70f054a4f419d7c33b4cd9845bd to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/C/9a83d70f054a4f419d7c33b4cd9845bd 2024-11-18T07:52:33,184 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/recovered.edits/562.seqid to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9/recovered.edits/562.seqid 2024-11-18T07:52:33,185 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1563b121b61f4db988d82be75c71ace9 2024-11-18T07:52:33,185 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T07:52:33,191 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:52:33,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-18T07:52:33,203 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T07:52:33,240 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T07:52:33,242 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:52:33,242 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T07:52:33,242 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731916353242"}]},"ts":"9223372036854775807"} 2024-11-18T07:52:33,246 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T07:52:33,247 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1563b121b61f4db988d82be75c71ace9, NAME => 'TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T07:52:33,247 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T07:52:33,247 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731916353247"}]},"ts":"9223372036854775807"} 2024-11-18T07:52:33,250 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T07:52:33,251 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:52:33,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 99 msec 2024-11-18T07:52:33,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-18T07:52:33,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T07:52:33,269 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:33,280 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;87fb3d252950:37987-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1820827738_22 at /127.0.0.1:42398 [Waiting for operation #260] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-349956365_22 at /127.0.0.1:41718 [Waiting for operation #133] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/87fb3d252950:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-349956365_22 at /127.0.0.1:42260 [Waiting for operation #276] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=666 (was 497) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=415 (was 980) 2024-11-18T07:52:33,290 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=666, ProcessCount=11, AvailableMemoryMB=415 2024-11-18T07:52:33,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T07:52:33,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T07:52:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T07:52:33,295 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T07:52:33,295 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:33,295 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-18T07:52:33,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-18T07:52:33,296 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T07:52:33,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741945_1121 (size=1061) 2024-11-18T07:52:33,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-18T07:52:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-18T07:52:33,705 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3 2024-11-18T07:52:33,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741946_1122 (size=53) 2024-11-18T07:52:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-18T07:52:34,112 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:52:34,112 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3f3b11bb24dc0fc107b3c002b8506511, disabling compactions & flushes 2024-11-18T07:52:34,112 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:34,112 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:34,112 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. after waiting 0 ms 2024-11-18T07:52:34,112 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:34,113 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:34,113 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3f3b11bb24dc0fc107b3c002b8506511: Waiting for close lock at 1731916354112Disabling compacts and flushes for region at 1731916354112Disabling writes for close at 1731916354112Writing region close event to WAL at 1731916354112Closed at 1731916354112 2024-11-18T07:52:34,114 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T07:52:34,115 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731916354114"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731916354114"}]},"ts":"1731916354114"} 2024-11-18T07:52:34,117 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T07:52:34,118 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T07:52:34,118 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916354118"}]},"ts":"1731916354118"} 2024-11-18T07:52:34,120 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T07:52:34,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f3b11bb24dc0fc107b3c002b8506511, ASSIGN}] 2024-11-18T07:52:34,122 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f3b11bb24dc0fc107b3c002b8506511, ASSIGN 2024-11-18T07:52:34,123 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f3b11bb24dc0fc107b3c002b8506511, ASSIGN; state=OFFLINE, location=87fb3d252950,37987,1731916323128; forceNewPlan=false, retain=false 2024-11-18T07:52:34,274 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=3f3b11bb24dc0fc107b3c002b8506511, regionState=OPENING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:52:34,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f3b11bb24dc0fc107b3c002b8506511, ASSIGN because future has completed 2024-11-18T07:52:34,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:52:34,431 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:34,431 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} 2024-11-18T07:52:34,431 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:34,432 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:52:34,432 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:34,432 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:34,433 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:34,434 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:52:34,435 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f3b11bb24dc0fc107b3c002b8506511 columnFamilyName A 2024-11-18T07:52:34,435 DEBUG [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:34,436 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.HStore(327): Store=3f3b11bb24dc0fc107b3c002b8506511/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:52:34,436 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:34,437 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:52:34,437 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f3b11bb24dc0fc107b3c002b8506511 columnFamilyName B 2024-11-18T07:52:34,437 DEBUG [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:34,438 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.HStore(327): Store=3f3b11bb24dc0fc107b3c002b8506511/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:52:34,438 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:34,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-18T07:52:34,439 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:52:34,439 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f3b11bb24dc0fc107b3c002b8506511 columnFamilyName C 2024-11-18T07:52:34,439 DEBUG [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:34,440 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.HStore(327): Store=3f3b11bb24dc0fc107b3c002b8506511/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:52:34,440 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:34,440 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:34,441 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:34,441 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:34,442 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:34,442 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:34,443 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T07:52:34,444 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:34,446 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T07:52:34,446 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 3f3b11bb24dc0fc107b3c002b8506511; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68339581, jitterRate=0.018339112401008606}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T07:52:34,446 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:34,447 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 3f3b11bb24dc0fc107b3c002b8506511: Running coprocessor pre-open hook at 1731916354432Writing region info on filesystem at 1731916354432Initializing all the Stores at 1731916354433 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916354433Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916354433Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916354433Cleaning up temporary data from old regions at 1731916354442 (+9 ms)Running coprocessor post-open hooks at 1731916354446 (+4 ms)Region opened successfully at 1731916354447 (+1 ms) 2024-11-18T07:52:34,448 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., pid=32, masterSystemTime=1731916354428 2024-11-18T07:52:34,450 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:34,450 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:34,451 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=3f3b11bb24dc0fc107b3c002b8506511, regionState=OPEN, openSeqNum=2, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:52:34,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:52:34,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-18T07:52:34,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 in 178 msec 2024-11-18T07:52:34,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-18T07:52:34,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f3b11bb24dc0fc107b3c002b8506511, ASSIGN in 336 msec 2024-11-18T07:52:34,460 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T07:52:34,461 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916354460"}]},"ts":"1731916354460"} 2024-11-18T07:52:34,462 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T07:52:34,464 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T07:52:34,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-11-18T07:52:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-18T07:52:35,459 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:35,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@769bdb6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:35,460 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:35,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:35,462 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279c0d05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:35,464 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13c68170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:35,466 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:35,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:35,470 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T07:52:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 87fb3d252950,42633,1731916322278 2024-11-18T07:52:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3accb36a 2024-11-18T07:52:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T07:52:35,472 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T07:52:35,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T07:52:35,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T07:52:35,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T07:52:35,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741947_1123 (size=1097) 2024-11-18T07:52:35,897 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-18T07:52:35,897 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-18T07:52:35,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T07:52:35,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f3b11bb24dc0fc107b3c002b8506511, REOPEN/MOVE}] 2024-11-18T07:52:35,911 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f3b11bb24dc0fc107b3c002b8506511, REOPEN/MOVE 2024-11-18T07:52:35,912 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=3f3b11bb24dc0fc107b3c002b8506511, regionState=CLOSING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:52:35,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f3b11bb24dc0fc107b3c002b8506511, REOPEN/MOVE because future has completed 2024-11-18T07:52:35,914 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T07:52:35,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:52:36,067 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,067 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T07:52:36,067 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 3f3b11bb24dc0fc107b3c002b8506511, disabling compactions & flushes 2024-11-18T07:52:36,067 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:36,067 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:36,067 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. after waiting 0 ms 2024-11-18T07:52:36,067 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:36,071 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T07:52:36,071 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:36,071 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 3f3b11bb24dc0fc107b3c002b8506511: Waiting for close lock at 1731916356067Running coprocessor pre-close hooks at 1731916356067Disabling compacts and flushes for region at 1731916356067Disabling writes for close at 1731916356067Writing region close event to WAL at 1731916356067Running coprocessor post-close hooks at 1731916356071 (+4 ms)Closed at 1731916356071 2024-11-18T07:52:36,072 WARN [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 3f3b11bb24dc0fc107b3c002b8506511 to self. 2024-11-18T07:52:36,074 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,074 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=3f3b11bb24dc0fc107b3c002b8506511, regionState=CLOSED 2024-11-18T07:52:36,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:52:36,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-18T07:52:36,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 in 162 msec 2024-11-18T07:52:36,079 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f3b11bb24dc0fc107b3c002b8506511, REOPEN/MOVE; state=CLOSED, location=87fb3d252950,37987,1731916323128; forceNewPlan=false, retain=true 2024-11-18T07:52:36,230 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=3f3b11bb24dc0fc107b3c002b8506511, regionState=OPENING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f3b11bb24dc0fc107b3c002b8506511, REOPEN/MOVE because future has completed 2024-11-18T07:52:36,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:52:36,389 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:36,389 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} 2024-11-18T07:52:36,390 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,390 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:52:36,390 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,390 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,395 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,396 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:52:36,402 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f3b11bb24dc0fc107b3c002b8506511 columnFamilyName A 2024-11-18T07:52:36,404 DEBUG [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:36,405 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.HStore(327): Store=3f3b11bb24dc0fc107b3c002b8506511/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:52:36,406 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,406 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:52:36,406 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f3b11bb24dc0fc107b3c002b8506511 columnFamilyName B 2024-11-18T07:52:36,406 DEBUG [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:36,407 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.HStore(327): Store=3f3b11bb24dc0fc107b3c002b8506511/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:52:36,407 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,408 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:52:36,408 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f3b11bb24dc0fc107b3c002b8506511 columnFamilyName C 2024-11-18T07:52:36,408 DEBUG [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:36,408 INFO [StoreOpener-3f3b11bb24dc0fc107b3c002b8506511-1 {}] regionserver.HStore(327): Store=3f3b11bb24dc0fc107b3c002b8506511/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:52:36,408 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:36,408 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,409 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,410 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,411 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,411 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,411 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T07:52:36,412 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,413 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 3f3b11bb24dc0fc107b3c002b8506511; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69371736, jitterRate=0.033719420433044434}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T07:52:36,413 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,415 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 3f3b11bb24dc0fc107b3c002b8506511: Running coprocessor pre-open hook at 1731916356390Writing region info on filesystem at 1731916356390Initializing all the Stores at 1731916356391 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916356391Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916356392 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916356392Cleaning up temporary data from old regions at 1731916356411 (+19 ms)Running coprocessor post-open hooks at 1731916356413 (+2 ms)Region opened successfully at 1731916356415 (+2 ms) 2024-11-18T07:52:36,416 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., pid=37, masterSystemTime=1731916356384 2024-11-18T07:52:36,417 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:36,417 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:36,418 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=3f3b11bb24dc0fc107b3c002b8506511, regionState=OPEN, openSeqNum=5, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:52:36,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-18T07:52:36,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 in 189 msec 2024-11-18T07:52:36,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-18T07:52:36,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f3b11bb24dc0fc107b3c002b8506511, REOPEN/MOVE in 513 msec 2024-11-18T07:52:36,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-18T07:52:36,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 525 msec 2024-11-18T07:52:36,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 951 msec 2024-11-18T07:52:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-18T07:52:36,435 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:36,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18f01ca4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:36,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:36,437 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df7409, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:36,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:36,440 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:36,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ba1da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a283555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,441 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:36,441 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:36,442 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b0f283e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:36,444 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65679af6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ba7f2f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,446 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:36,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:36,447 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ecec32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:36,449 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:36,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f8b3876, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14984fac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,450 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:36,450 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:36,451 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:36,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:36,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:36,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25cf638, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:36,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:36,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:36,453 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@707b33a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e890ec6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,458 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:36,459 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:36,459 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@550b18fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:36,462 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6faf0444, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f62ef44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:36,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:36,465 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33b1bc25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:36,466 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7287850a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f60d5da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,471 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:36,471 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:36,472 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27005e06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:36,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:36,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:36,474 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:36,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341602f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@139f617f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,475 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:36,475 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:36,476 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aa9c030, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:36,478 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:36,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e57bfd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7def024f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,480 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:52:36,480 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:52:36,481 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:52:36,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:52:36,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:52:36,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dffd523, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:52:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:52:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:36,483 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:52:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b6c54a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:52:36,485 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:36,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:52:36,486 DEBUG [Thread-621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:36,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-18T07:52:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:36,487 DEBUG [Thread-623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T07:52:36,488 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:52:36,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:36,488 DEBUG [Thread-629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:36,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:36,489 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:52:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:36,489 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:36,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:52:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:36,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:36,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:36,491 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:36,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:36,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:36,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:52:36,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:36,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:36,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:36,495 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:52:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:52:36,497 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:36,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:36,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:52:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:52:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:52:36,499 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:36,501 DEBUG [Thread-627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:36,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:52:36,502 DEBUG [Thread-632 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:52:36,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:36,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:36,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:52:36,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:52:36,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:36,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:52:36,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:52:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:52:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:52:36,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:36,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:36,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:36,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:36,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:36,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:36,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b762c385198149ed9adfc1c9ce8c6873_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916356507/Put/seqid=0 2024-11-18T07:52:36,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731916356537, max timestamp=1731916356566Num uniques 859; 2024-11-18T07:52:36,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T07:52:36,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731916356537, max timestamp=1731916356566Num uniques 859; 2024-11-18T07:52:36,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T07:52:36,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731916356537, max timestamp=1731916356566Num uniques 859; 2024-11-18T07:52:36,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T07:52:36,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741948_1124 (size=17034) 2024-11-18T07:52:36,601 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T07:52:36,608 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b762c385198149ed9adfc1c9ce8c6873_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b762c385198149ed9adfc1c9ce8c6873_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:36,610 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/da4c05bf64724fc9b326aa78d05650d6, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:36,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/da4c05bf64724fc9b326aa78d05650d6 is 175, key is test_row_0/A:col10/1731916356507/Put/seqid=0 2024-11-18T07:52:36,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916416609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916416611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916416620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:36,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916416628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916416630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:36,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T07:52:36,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:36,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:36,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:36,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:36,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:36,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741949_1125 (size=48139) 2024-11-18T07:52:36,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916416740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:36,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916416740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916416740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:36,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916416740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916416741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:36,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T07:52:36,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:36,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:36,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:36,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:36,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T07:52:36,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T07:52:36,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:36,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:36,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:36,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:36,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:36,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:36,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916416951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:36,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916416958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916416958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:36,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:36,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916416962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:36,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916416962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,068 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/da4c05bf64724fc9b326aa78d05650d6 2024-11-18T07:52:37,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T07:52:37,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:37,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:37,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:37,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/c3033bc0fae64daba9ea06132cf3aa97 is 50, key is test_row_0/B:col10/1731916356507/Put/seqid=0 2024-11-18T07:52:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T07:52:37,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741950_1126 (size=12001) 2024-11-18T07:52:37,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/c3033bc0fae64daba9ea06132cf3aa97 2024-11-18T07:52:37,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/bef40946d3e64ed681f2d1aa9aa4cdd0 is 50, key is test_row_0/C:col10/1731916356507/Put/seqid=0 2024-11-18T07:52:37,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741951_1127 (size=12001) 2024-11-18T07:52:37,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/bef40946d3e64ed681f2d1aa9aa4cdd0 2024-11-18T07:52:37,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/da4c05bf64724fc9b326aa78d05650d6 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/da4c05bf64724fc9b326aa78d05650d6 2024-11-18T07:52:37,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/da4c05bf64724fc9b326aa78d05650d6, entries=250, sequenceid=15, filesize=47.0 K 2024-11-18T07:52:37,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/c3033bc0fae64daba9ea06132cf3aa97 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/c3033bc0fae64daba9ea06132cf3aa97 2024-11-18T07:52:37,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/c3033bc0fae64daba9ea06132cf3aa97, entries=150, sequenceid=15, filesize=11.7 K 2024-11-18T07:52:37,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/bef40946d3e64ed681f2d1aa9aa4cdd0 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/bef40946d3e64ed681f2d1aa9aa4cdd0 2024-11-18T07:52:37,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T07:52:37,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:37,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:37,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:37,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/bef40946d3e64ed681f2d1aa9aa4cdd0, entries=150, sequenceid=15, filesize=11.7 K 2024-11-18T07:52:37,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 3f3b11bb24dc0fc107b3c002b8506511 in 758ms, sequenceid=15, compaction requested=false 2024-11-18T07:52:37,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:37,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-18T07:52:37,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:37,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:37,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:37,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:37,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:37,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:37,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916417288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916417289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:37,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916417290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916417301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:37,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916417303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:37,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118049e8d85d01f4321b5328f12f2d1b4c8_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916357281/Put/seqid=0 2024-11-18T07:52:37,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741952_1128 (size=12154) 2024-11-18T07:52:37,330 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:37,338 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118049e8d85d01f4321b5328f12f2d1b4c8_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118049e8d85d01f4321b5328f12f2d1b4c8_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:37,340 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/5f9bcb5614bc4cf691353c3ad8a98bc2, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:37,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/5f9bcb5614bc4cf691353c3ad8a98bc2 is 175, key is test_row_0/A:col10/1731916357281/Put/seqid=0 2024-11-18T07:52:37,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741953_1129 (size=30955) 2024-11-18T07:52:37,369 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/5f9bcb5614bc4cf691353c3ad8a98bc2 2024-11-18T07:52:37,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/31989abfd61f4d0e8aaafca3e9b6d511 is 50, key is test_row_0/B:col10/1731916357281/Put/seqid=0 2024-11-18T07:52:37,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:37,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T07:52:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916417408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:37,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:37,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916417409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:37,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:37,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:37,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741954_1130 (size=12001) 2024-11-18T07:52:37,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916417419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916417420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:37,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916417421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:37,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T07:52:37,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:37,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:37,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:37,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T07:52:37,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916417628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:37,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916417629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:37,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916417629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916417629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916417638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T07:52:37,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:37,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:37,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:37,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/31989abfd61f4d0e8aaafca3e9b6d511 2024-11-18T07:52:37,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/027e21a481264f4ead3d2046764406ed is 50, key is test_row_0/C:col10/1731916357281/Put/seqid=0 2024-11-18T07:52:37,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741955_1131 (size=12001) 2024-11-18T07:52:37,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/027e21a481264f4ead3d2046764406ed 2024-11-18T07:52:37,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/5f9bcb5614bc4cf691353c3ad8a98bc2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/5f9bcb5614bc4cf691353c3ad8a98bc2 2024-11-18T07:52:37,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T07:52:37,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:37,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:37,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:37,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:37,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/5f9bcb5614bc4cf691353c3ad8a98bc2, entries=150, sequenceid=51, filesize=30.2 K 2024-11-18T07:52:37,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/31989abfd61f4d0e8aaafca3e9b6d511 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/31989abfd61f4d0e8aaafca3e9b6d511 2024-11-18T07:52:37,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/31989abfd61f4d0e8aaafca3e9b6d511, entries=150, sequenceid=51, filesize=11.7 K 2024-11-18T07:52:37,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/027e21a481264f4ead3d2046764406ed as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/027e21a481264f4ead3d2046764406ed 2024-11-18T07:52:37,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/027e21a481264f4ead3d2046764406ed, entries=150, sequenceid=51, filesize=11.7 K 2024-11-18T07:52:37,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 3f3b11bb24dc0fc107b3c002b8506511 in 649ms, sequenceid=51, compaction requested=false 2024-11-18T07:52:37,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:37,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T07:52:37,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:37,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:37,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:37,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:37,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:37,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:37,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111843aabe3429aa411396daaca279033a3d_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916357287/Put/seqid=0 2024-11-18T07:52:37,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916417976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:37,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731916357961, max timestamp=1731916357982Num uniques 859; 2024-11-18T07:52:37,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-18T07:52:37,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:37,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:37,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731916357961, max timestamp=1731916357982Num uniques 859; 2024-11-18T07:52:37,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-18T07:52:37,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731916357961, max timestamp=1731916357982Num uniques 859; 2024-11-18T07:52:37,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-18T07:52:37,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741956_1132 (size=17034) 2024-11-18T07:52:38,002 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916418007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916418009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916418011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,018 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111843aabe3429aa411396daaca279033a3d_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111843aabe3429aa411396daaca279033a3d_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:38,020 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/90c1225a8f0e406bb4c87f035424be07, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:38,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916418014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/90c1225a8f0e406bb4c87f035424be07 is 175, key is test_row_0/A:col10/1731916357287/Put/seqid=0 2024-11-18T07:52:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741957_1133 (size=48139) 2024-11-18T07:52:38,036 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/90c1225a8f0e406bb4c87f035424be07 2024-11-18T07:52:38,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T07:52:38,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:38,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/a0ec93e408744d3db84cf0b516472ab3 is 50, key is test_row_0/B:col10/1731916357287/Put/seqid=0 2024-11-18T07:52:38,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741958_1134 (size=12001) 2024-11-18T07:52:38,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/a0ec93e408744d3db84cf0b516472ab3 2024-11-18T07:52:38,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916418098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/1f30b7fca59a4acdba430a296e636ce4 is 50, key is test_row_0/C:col10/1731916357287/Put/seqid=0 2024-11-18T07:52:38,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916418120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916418129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916418129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916418129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741959_1135 (size=12001) 2024-11-18T07:52:38,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/1f30b7fca59a4acdba430a296e636ce4 2024-11-18T07:52:38,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/90c1225a8f0e406bb4c87f035424be07 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/90c1225a8f0e406bb4c87f035424be07 2024-11-18T07:52:38,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/90c1225a8f0e406bb4c87f035424be07, entries=250, sequenceid=62, filesize=47.0 K 2024-11-18T07:52:38,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T07:52:38,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:38,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/a0ec93e408744d3db84cf0b516472ab3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/a0ec93e408744d3db84cf0b516472ab3 2024-11-18T07:52:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,202 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T07:52:38,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/a0ec93e408744d3db84cf0b516472ab3, entries=150, sequenceid=62, filesize=11.7 K 2024-11-18T07:52:38,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/1f30b7fca59a4acdba430a296e636ce4 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/1f30b7fca59a4acdba430a296e636ce4 2024-11-18T07:52:38,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/1f30b7fca59a4acdba430a296e636ce4, entries=150, sequenceid=62, filesize=11.7 K 2024-11-18T07:52:38,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 3f3b11bb24dc0fc107b3c002b8506511 in 293ms, sequenceid=62, compaction requested=true 2024-11-18T07:52:38,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:38,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:38,236 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:38,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:38,236 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:38,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:38,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:38,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:38,237 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:38,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:38,237 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/A is initiating minor compaction (all files) 2024-11-18T07:52:38,238 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/A in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,238 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/da4c05bf64724fc9b326aa78d05650d6, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/5f9bcb5614bc4cf691353c3ad8a98bc2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/90c1225a8f0e406bb4c87f035424be07] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=124.3 K 2024-11-18T07:52:38,238 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,238 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/da4c05bf64724fc9b326aa78d05650d6, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/5f9bcb5614bc4cf691353c3ad8a98bc2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/90c1225a8f0e406bb4c87f035424be07] 2024-11-18T07:52:38,238 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:38,239 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/B is initiating minor compaction (all files) 2024-11-18T07:52:38,239 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/B in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,239 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/c3033bc0fae64daba9ea06132cf3aa97, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/31989abfd61f4d0e8aaafca3e9b6d511, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/a0ec93e408744d3db84cf0b516472ab3] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=35.2 K 2024-11-18T07:52:38,239 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting da4c05bf64724fc9b326aa78d05650d6, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731916356502 2024-11-18T07:52:38,240 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f9bcb5614bc4cf691353c3ad8a98bc2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731916356602 2024-11-18T07:52:38,240 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting c3033bc0fae64daba9ea06132cf3aa97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731916356503 2024-11-18T07:52:38,242 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90c1225a8f0e406bb4c87f035424be07, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731916357287 2024-11-18T07:52:38,242 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 31989abfd61f4d0e8aaafca3e9b6d511, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731916356602 2024-11-18T07:52:38,243 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting a0ec93e408744d3db84cf0b516472ab3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731916357287 2024-11-18T07:52:38,253 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#B#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:38,253 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/2b60ba4b7fde4fda93ed64e202ebfc06 is 50, key is test_row_0/B:col10/1731916357287/Put/seqid=0 2024-11-18T07:52:38,263 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:38,276 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118ab41149869cf4bc8a458fb6e9ae851ad_3f3b11bb24dc0fc107b3c002b8506511 store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:38,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741960_1136 (size=12104) 2024-11-18T07:52:38,283 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118ab41149869cf4bc8a458fb6e9ae851ad_3f3b11bb24dc0fc107b3c002b8506511, store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:38,283 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ab41149869cf4bc8a458fb6e9ae851ad_3f3b11bb24dc0fc107b3c002b8506511 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:38,289 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/2b60ba4b7fde4fda93ed64e202ebfc06 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/2b60ba4b7fde4fda93ed64e202ebfc06 2024-11-18T07:52:38,295 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/B of 3f3b11bb24dc0fc107b3c002b8506511 into 2b60ba4b7fde4fda93ed64e202ebfc06(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:38,295 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:38,295 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/B, priority=13, startTime=1731916358236; duration=0sec 2024-11-18T07:52:38,295 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:38,295 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:B 2024-11-18T07:52:38,295 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:38,296 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:38,296 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/C is initiating minor compaction (all files) 2024-11-18T07:52:38,296 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/C in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,297 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/bef40946d3e64ed681f2d1aa9aa4cdd0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/027e21a481264f4ead3d2046764406ed, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/1f30b7fca59a4acdba430a296e636ce4] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=35.2 K 2024-11-18T07:52:38,298 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting bef40946d3e64ed681f2d1aa9aa4cdd0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731916356503 2024-11-18T07:52:38,298 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 027e21a481264f4ead3d2046764406ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731916356602 2024-11-18T07:52:38,299 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f30b7fca59a4acdba430a296e636ce4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731916357287 2024-11-18T07:52:38,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741961_1137 (size=4469) 2024-11-18T07:52:38,317 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#C#compaction#119 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:38,318 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/d3b2afb6ae3a45489de5707c29ee4646 is 50, key is test_row_0/C:col10/1731916357287/Put/seqid=0 2024-11-18T07:52:38,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741962_1138 (size=12104) 2024-11-18T07:52:38,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=140.89 KB heapSize=381.47 KB 2024-11-18T07:52:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:38,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T07:52:38,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:38,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916418350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916418354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916418348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916418362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185b06fdcb7c4345f28cca7989d85b2642_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916358004/Put/seqid=0 2024-11-18T07:52:38,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741963_1139 (size=14594) 2024-11-18T07:52:38,387 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,394 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185b06fdcb7c4345f28cca7989d85b2642_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185b06fdcb7c4345f28cca7989d85b2642_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:38,395 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/0c1d8274bb354362a119ed503cc9f4e9, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:38,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/0c1d8274bb354362a119ed503cc9f4e9 is 175, key is test_row_0/A:col10/1731916358004/Put/seqid=0 2024-11-18T07:52:38,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916418389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741964_1140 (size=39549) 2024-11-18T07:52:38,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916418478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916418478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916418478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916418478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T07:52:38,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:38,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916418508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T07:52:38,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T07:52:38,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:38,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916418688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916418688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916418698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916418698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,705 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#A#compaction#118 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:38,705 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/65acce3b2f6243f394b4d40beceb8333 is 175, key is test_row_0/A:col10/1731916357287/Put/seqid=0 2024-11-18T07:52:38,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741965_1141 (size=31058) 2024-11-18T07:52:38,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916418718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:38,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:38,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:38,731 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/65acce3b2f6243f394b4d40beceb8333 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/65acce3b2f6243f394b4d40beceb8333 2024-11-18T07:52:38,740 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/d3b2afb6ae3a45489de5707c29ee4646 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/d3b2afb6ae3a45489de5707c29ee4646 2024-11-18T07:52:38,743 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/A of 3f3b11bb24dc0fc107b3c002b8506511 into 65acce3b2f6243f394b4d40beceb8333(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:38,743 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:38,743 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/A, priority=13, startTime=1731916358236; duration=0sec 2024-11-18T07:52:38,743 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:38,743 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:A 2024-11-18T07:52:38,753 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/C of 3f3b11bb24dc0fc107b3c002b8506511 into d3b2afb6ae3a45489de5707c29ee4646(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:38,753 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:38,753 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/C, priority=13, startTime=1731916358237; duration=0sec 2024-11-18T07:52:38,753 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:38,753 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:C 2024-11-18T07:52:38,814 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/0c1d8274bb354362a119ed503cc9f4e9 2024-11-18T07:52:38,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T07:52:38,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:38,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:38,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/426f99ec4e5848688477da001f7a92b0 is 50, key is test_row_0/B:col10/1731916358004/Put/seqid=0 2024-11-18T07:52:38,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741966_1142 (size=12001) 2024-11-18T07:52:38,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/426f99ec4e5848688477da001f7a92b0 2024-11-18T07:52:38,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/ed6c3752a8c549fdaf9b640a3beaf503 is 50, key is test_row_0/C:col10/1731916358004/Put/seqid=0 2024-11-18T07:52:38,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741967_1143 (size=12001) 2024-11-18T07:52:38,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/ed6c3752a8c549fdaf9b640a3beaf503 2024-11-18T07:52:38,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/0c1d8274bb354362a119ed503cc9f4e9 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/0c1d8274bb354362a119ed503cc9f4e9 2024-11-18T07:52:38,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/0c1d8274bb354362a119ed503cc9f4e9, entries=200, sequenceid=96, filesize=38.6 K 2024-11-18T07:52:38,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/426f99ec4e5848688477da001f7a92b0 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/426f99ec4e5848688477da001f7a92b0 2024-11-18T07:52:38,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/426f99ec4e5848688477da001f7a92b0, entries=150, sequenceid=96, filesize=11.7 K 2024-11-18T07:52:38,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/ed6c3752a8c549fdaf9b640a3beaf503 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/ed6c3752a8c549fdaf9b640a3beaf503 2024-11-18T07:52:38,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/ed6c3752a8c549fdaf9b640a3beaf503, entries=150, sequenceid=96, filesize=11.7 K 2024-11-18T07:52:38,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~381.42 KB/390576, currentSize=53.67 KB/54960 for 3f3b11bb24dc0fc107b3c002b8506511 in 628ms, sequenceid=96, compaction requested=false 2024-11-18T07:52:38,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T07:52:38,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,971 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T07:52:38,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:38,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:38,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:38,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111824ad543dad564d5588fb8e81e34dff72_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916358358/Put/seqid=0 2024-11-18T07:52:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741968_1144 (size=9714) 2024-11-18T07:52:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:39,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916359036, max timestamp=1731916359050Num uniques 859; 2024-11-18T07:52:39,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:52:39,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916359036, max timestamp=1731916359050Num uniques 859; 2024-11-18T07:52:39,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:52:39,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916359036, max timestamp=1731916359050Num uniques 859; 2024-11-18T07:52:39,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:52:39,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916419084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916419085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916419086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916419087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916419097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916419208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916419208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916419209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916419209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916419209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:39,402 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111824ad543dad564d5588fb8e81e34dff72_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111824ad543dad564d5588fb8e81e34dff72_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:39,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/d1168931e4dd43468c715f706f8708de, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:39,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/d1168931e4dd43468c715f706f8708de is 175, key is test_row_0/A:col10/1731916358358/Put/seqid=0 2024-11-18T07:52:39,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916419419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916419420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916419420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916419421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916419421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741969_1145 (size=22361) 2024-11-18T07:52:39,440 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/d1168931e4dd43468c715f706f8708de 2024-11-18T07:52:39,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/b7326b919cdd40b899038a4c452a927f is 50, key is test_row_0/B:col10/1731916358358/Put/seqid=0 2024-11-18T07:52:39,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741970_1146 (size=9657) 2024-11-18T07:52:39,465 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/b7326b919cdd40b899038a4c452a927f 2024-11-18T07:52:39,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/0e0da02e1cb749fbaeaed57686cfdefe is 50, key is test_row_0/C:col10/1731916358358/Put/seqid=0 2024-11-18T07:52:39,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741971_1147 (size=9657) 2024-11-18T07:52:39,508 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/0e0da02e1cb749fbaeaed57686cfdefe 2024-11-18T07:52:39,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/d1168931e4dd43468c715f706f8708de as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/d1168931e4dd43468c715f706f8708de 2024-11-18T07:52:39,522 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/d1168931e4dd43468c715f706f8708de, entries=100, sequenceid=109, filesize=21.8 K 2024-11-18T07:52:39,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/b7326b919cdd40b899038a4c452a927f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/b7326b919cdd40b899038a4c452a927f 2024-11-18T07:52:39,533 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/b7326b919cdd40b899038a4c452a927f, entries=100, sequenceid=109, filesize=9.4 K 2024-11-18T07:52:39,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/0e0da02e1cb749fbaeaed57686cfdefe as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0e0da02e1cb749fbaeaed57686cfdefe 2024-11-18T07:52:39,544 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0e0da02e1cb749fbaeaed57686cfdefe, entries=100, sequenceid=109, filesize=9.4 K 2024-11-18T07:52:39,545 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 3f3b11bb24dc0fc107b3c002b8506511 in 574ms, sequenceid=109, compaction requested=true 2024-11-18T07:52:39,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:39,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:39,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-18T07:52:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-18T07:52:39,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-18T07:52:39,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0590 sec 2024-11-18T07:52:39,554 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 3.0670 sec 2024-11-18T07:52:39,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-11-18T07:52:39,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:39,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:39,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:39,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:39,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:39,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:39,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187c6984d2b29849b1a629403057a7a4c6_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916359072/Put/seqid=0 2024-11-18T07:52:39,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916419750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916419752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916419759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916419761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916419761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741972_1148 (size=12304) 2024-11-18T07:52:39,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916419869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916419869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916419878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916419878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:39,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916419878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:39,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:40,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916420078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:40,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916420088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916420088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:40,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:40,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916420088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:40,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916420089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:40,198 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:40,203 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187c6984d2b29849b1a629403057a7a4c6_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187c6984d2b29849b1a629403057a7a4c6_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:40,204 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/83e0361b33e14ad09a504e4bed61cc10, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:40,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/83e0361b33e14ad09a504e4bed61cc10 is 175, key is test_row_0/A:col10/1731916359072/Put/seqid=0 2024-11-18T07:52:40,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741973_1149 (size=31105) 2024-11-18T07:52:40,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916420398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:40,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916420398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916420398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:40,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916420398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:40,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916420398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:40,610 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/83e0361b33e14ad09a504e4bed61cc10 2024-11-18T07:52:40,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/07e5b87d8ba64dbb829baac8f7feedbb is 50, key is test_row_0/B:col10/1731916359072/Put/seqid=0 2024-11-18T07:52:40,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741974_1150 (size=12151) 2024-11-18T07:52:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T07:52:40,648 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:40,649 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:52:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-18T07:52:40,653 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:52:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T07:52:40,653 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:52:40,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:52:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T07:52:40,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T07:52:40,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:40,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:40,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:40,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:40,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:40,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916420918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:40,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916420920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:40,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:40,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916420921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:40,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916420921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:40,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916420921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:40,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T07:52:40,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:40,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:40,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:40,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:40,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:40,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:40,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T07:52:41,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/07e5b87d8ba64dbb829baac8f7feedbb 2024-11-18T07:52:41,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/f8dace3717cd4113b28bad9057453c10 is 50, key is test_row_0/C:col10/1731916359072/Put/seqid=0 2024-11-18T07:52:41,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741975_1151 (size=12151) 2024-11-18T07:52:41,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/f8dace3717cd4113b28bad9057453c10 2024-11-18T07:52:41,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/83e0361b33e14ad09a504e4bed61cc10 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/83e0361b33e14ad09a504e4bed61cc10 2024-11-18T07:52:41,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/83e0361b33e14ad09a504e4bed61cc10, entries=150, sequenceid=145, filesize=30.4 K 2024-11-18T07:52:41,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/07e5b87d8ba64dbb829baac8f7feedbb as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/07e5b87d8ba64dbb829baac8f7feedbb 2024-11-18T07:52:41,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T07:52:41,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:41,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:41,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:41,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:41,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:41,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:41,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/07e5b87d8ba64dbb829baac8f7feedbb, entries=150, sequenceid=145, filesize=11.9 K 2024-11-18T07:52:41,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/f8dace3717cd4113b28bad9057453c10 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/f8dace3717cd4113b28bad9057453c10 2024-11-18T07:52:41,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/f8dace3717cd4113b28bad9057453c10, entries=150, sequenceid=145, filesize=11.9 K 2024-11-18T07:52:41,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=60.38 KB/61830 for 3f3b11bb24dc0fc107b3c002b8506511 in 1407ms, sequenceid=145, compaction requested=true 2024-11-18T07:52:41,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:41,145 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:41,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:41,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:41,146 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:41,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:41,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:41,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:41,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:41,148 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124073 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:41,148 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/A is initiating minor compaction (all files) 2024-11-18T07:52:41,148 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/A in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:41,148 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/65acce3b2f6243f394b4d40beceb8333, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/0c1d8274bb354362a119ed503cc9f4e9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/d1168931e4dd43468c715f706f8708de, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/83e0361b33e14ad09a504e4bed61cc10] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=121.2 K 2024-11-18T07:52:41,148 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:41,148 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/65acce3b2f6243f394b4d40beceb8333, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/0c1d8274bb354362a119ed503cc9f4e9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/d1168931e4dd43468c715f706f8708de, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/83e0361b33e14ad09a504e4bed61cc10] 2024-11-18T07:52:41,151 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65acce3b2f6243f394b4d40beceb8333, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731916357287 2024-11-18T07:52:41,151 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:41,151 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/B is initiating minor compaction (all files) 2024-11-18T07:52:41,151 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/B in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:41,152 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/2b60ba4b7fde4fda93ed64e202ebfc06, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/426f99ec4e5848688477da001f7a92b0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/b7326b919cdd40b899038a4c452a927f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/07e5b87d8ba64dbb829baac8f7feedbb] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=44.8 K 2024-11-18T07:52:41,152 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c1d8274bb354362a119ed503cc9f4e9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731916358004 2024-11-18T07:52:41,154 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b60ba4b7fde4fda93ed64e202ebfc06, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731916357287 2024-11-18T07:52:41,154 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1168931e4dd43468c715f706f8708de, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731916358349 2024-11-18T07:52:41,154 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 426f99ec4e5848688477da001f7a92b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731916358004 2024-11-18T07:52:41,154 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83e0361b33e14ad09a504e4bed61cc10, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731916359072 2024-11-18T07:52:41,155 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting b7326b919cdd40b899038a4c452a927f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731916358349 2024-11-18T07:52:41,156 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 07e5b87d8ba64dbb829baac8f7feedbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731916359072 2024-11-18T07:52:41,168 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:41,179 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#B#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:41,179 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/c273cad045eb45918c1b852161e30192 is 50, key is test_row_0/B:col10/1731916359072/Put/seqid=0 2024-11-18T07:52:41,180 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118b72def8c176c474ebec906140b5251e9_3f3b11bb24dc0fc107b3c002b8506511 store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:41,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741976_1152 (size=12391) 2024-11-18T07:52:41,189 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118b72def8c176c474ebec906140b5251e9_3f3b11bb24dc0fc107b3c002b8506511, store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:41,189 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b72def8c176c474ebec906140b5251e9_3f3b11bb24dc0fc107b3c002b8506511 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:41,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741977_1153 (size=4469) 2024-11-18T07:52:41,202 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#A#compaction#129 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:41,202 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/acea7b8087b44ba280bf0d7ceb86301b is 175, key is test_row_0/A:col10/1731916359072/Put/seqid=0 2024-11-18T07:52:41,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741978_1154 (size=31345) 2024-11-18T07:52:41,225 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/acea7b8087b44ba280bf0d7ceb86301b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/acea7b8087b44ba280bf0d7ceb86301b 2024-11-18T07:52:41,237 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/A of 3f3b11bb24dc0fc107b3c002b8506511 into acea7b8087b44ba280bf0d7ceb86301b(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:41,237 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:41,237 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/A, priority=12, startTime=1731916361145; duration=0sec 2024-11-18T07:52:41,237 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:41,237 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:A 2024-11-18T07:52:41,237 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:41,243 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:41,243 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/C is initiating minor compaction (all files) 2024-11-18T07:52:41,243 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/C in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:41,244 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/d3b2afb6ae3a45489de5707c29ee4646, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/ed6c3752a8c549fdaf9b640a3beaf503, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0e0da02e1cb749fbaeaed57686cfdefe, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/f8dace3717cd4113b28bad9057453c10] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=44.8 K 2024-11-18T07:52:41,244 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3b2afb6ae3a45489de5707c29ee4646, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731916357287 2024-11-18T07:52:41,245 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed6c3752a8c549fdaf9b640a3beaf503, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731916358004 2024-11-18T07:52:41,245 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e0da02e1cb749fbaeaed57686cfdefe, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731916358349 2024-11-18T07:52:41,245 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8dace3717cd4113b28bad9057453c10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731916359072 2024-11-18T07:52:41,266 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#C#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:41,267 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/1ccfc07f1ea9474fa45d4e00341999c6 is 50, key is test_row_0/C:col10/1731916359072/Put/seqid=0 2024-11-18T07:52:41,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741979_1155 (size=12391) 2024-11-18T07:52:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T07:52:41,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T07:52:41,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:41,281 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T07:52:41,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:41,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:41,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:41,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:41,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:41,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:41,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118be12e8efac74453e8e62124f16f7fb6f_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916359757/Put/seqid=0 2024-11-18T07:52:41,302 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/1ccfc07f1ea9474fa45d4e00341999c6 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/1ccfc07f1ea9474fa45d4e00341999c6 2024-11-18T07:52:41,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741980_1156 (size=12304) 2024-11-18T07:52:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:41,312 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/C of 3f3b11bb24dc0fc107b3c002b8506511 into 1ccfc07f1ea9474fa45d4e00341999c6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:41,312 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:41,312 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/C, priority=12, startTime=1731916361147; duration=0sec 2024-11-18T07:52:41,312 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:41,313 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:C 2024-11-18T07:52:41,318 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118be12e8efac74453e8e62124f16f7fb6f_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118be12e8efac74453e8e62124f16f7fb6f_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:41,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/97fe9bb4056c4891bb382c585de16861, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:41,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/97fe9bb4056c4891bb382c585de16861 is 175, key is test_row_0/A:col10/1731916359757/Put/seqid=0 2024-11-18T07:52:41,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741981_1157 (size=31105) 2024-11-18T07:52:41,333 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/97fe9bb4056c4891bb382c585de16861 2024-11-18T07:52:41,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/53c6f1a50a0e442eb5630fdd60173017 is 50, key is test_row_0/B:col10/1731916359757/Put/seqid=0 2024-11-18T07:52:41,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741982_1158 (size=12151) 2024-11-18T07:52:41,593 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/c273cad045eb45918c1b852161e30192 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/c273cad045eb45918c1b852161e30192 2024-11-18T07:52:41,601 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/B of 3f3b11bb24dc0fc107b3c002b8506511 into c273cad045eb45918c1b852161e30192(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:41,601 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:41,601 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/B, priority=12, startTime=1731916361146; duration=0sec 2024-11-18T07:52:41,601 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:41,601 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:B 2024-11-18T07:52:41,762 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/53c6f1a50a0e442eb5630fdd60173017 2024-11-18T07:52:41,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/8d103e618f53481c895c071945d50703 is 50, key is test_row_0/C:col10/1731916359757/Put/seqid=0 2024-11-18T07:52:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T07:52:41,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741983_1159 (size=12151) 2024-11-18T07:52:41,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:41,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916421964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:41,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916421965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:41,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916421967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:41,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916421968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:41,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916421970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:41,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:41,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:41,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916422078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916422078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916422078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916422078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916422078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,190 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/8d103e618f53481c895c071945d50703 2024-11-18T07:52:42,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/97fe9bb4056c4891bb382c585de16861 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/97fe9bb4056c4891bb382c585de16861 2024-11-18T07:52:42,203 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/97fe9bb4056c4891bb382c585de16861, entries=150, sequenceid=158, filesize=30.4 K 2024-11-18T07:52:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/53c6f1a50a0e442eb5630fdd60173017 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/53c6f1a50a0e442eb5630fdd60173017 2024-11-18T07:52:42,249 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/53c6f1a50a0e442eb5630fdd60173017, entries=150, sequenceid=158, filesize=11.9 K 2024-11-18T07:52:42,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/8d103e618f53481c895c071945d50703 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/8d103e618f53481c895c071945d50703 2024-11-18T07:52:42,257 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/8d103e618f53481c895c071945d50703, entries=150, sequenceid=158, filesize=11.9 K 2024-11-18T07:52:42,259 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 3f3b11bb24dc0fc107b3c002b8506511 in 978ms, sequenceid=158, compaction requested=false 2024-11-18T07:52:42,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:42,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:42,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-18T07:52:42,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-18T07:52:42,264 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-18T07:52:42,264 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6080 sec 2024-11-18T07:52:42,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.6170 sec 2024-11-18T07:52:42,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:42,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-18T07:52:42,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:42,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:42,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:42,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:42,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:42,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:42,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d5ff3d29c4b44aaf9319ca32e68c3975_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916362290/Put/seqid=0 2024-11-18T07:52:42,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916422303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916422312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916422313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916422322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916422323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741984_1160 (size=12304) 2024-11-18T07:52:42,356 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,362 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d5ff3d29c4b44aaf9319ca32e68c3975_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d5ff3d29c4b44aaf9319ca32e68c3975_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:42,364 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/1fa21e47b7924dd98d07b4919970c7d3, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:42,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/1fa21e47b7924dd98d07b4919970c7d3 is 175, key is test_row_0/A:col10/1731916362290/Put/seqid=0 2024-11-18T07:52:42,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741985_1161 (size=31105) 2024-11-18T07:52:42,373 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/1fa21e47b7924dd98d07b4919970c7d3 2024-11-18T07:52:42,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/8d38a5786a63441aa0b914d86c802ece is 50, key is test_row_0/B:col10/1731916362290/Put/seqid=0 2024-11-18T07:52:42,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741986_1162 (size=12151) 2024-11-18T07:52:42,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916422418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916422427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916422428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916422428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916422428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916422628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916422638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916422638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916422638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916422638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:42,725 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T07:52:42,725 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-18T07:52:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T07:52:42,805 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:42,806 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:52:42,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-18T07:52:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T07:52:42,810 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:52:42,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/8d38a5786a63441aa0b914d86c802ece 2024-11-18T07:52:42,815 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:52:42,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:52:42,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/9af01dde2e2d4e70b41388bcd8901109 is 50, key is test_row_0/C:col10/1731916362290/Put/seqid=0 2024-11-18T07:52:42,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741987_1163 (size=12151) 2024-11-18T07:52:42,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/9af01dde2e2d4e70b41388bcd8901109 2024-11-18T07:52:42,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/1fa21e47b7924dd98d07b4919970c7d3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/1fa21e47b7924dd98d07b4919970c7d3 2024-11-18T07:52:42,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/1fa21e47b7924dd98d07b4919970c7d3, entries=150, sequenceid=182, filesize=30.4 K 2024-11-18T07:52:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T07:52:42,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/8d38a5786a63441aa0b914d86c802ece as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/8d38a5786a63441aa0b914d86c802ece 2024-11-18T07:52:42,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/8d38a5786a63441aa0b914d86c802ece, entries=150, sequenceid=182, filesize=11.9 K 2024-11-18T07:52:42,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/9af01dde2e2d4e70b41388bcd8901109 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/9af01dde2e2d4e70b41388bcd8901109 2024-11-18T07:52:42,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/9af01dde2e2d4e70b41388bcd8901109, entries=150, sequenceid=182, filesize=11.9 K 2024-11-18T07:52:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for 3f3b11bb24dc0fc107b3c002b8506511 in 650ms, sequenceid=182, compaction requested=true 2024-11-18T07:52:42,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:42,941 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:42,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:42,942 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:42,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:42,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:42,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:42,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,943 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:42,943 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,946 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/B is initiating minor compaction (all files) 2024-11-18T07:52:42,946 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/B in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:42,946 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/c273cad045eb45918c1b852161e30192, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/53c6f1a50a0e442eb5630fdd60173017, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/8d38a5786a63441aa0b914d86c802ece] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=35.8 K 2024-11-18T07:52:42,946 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/A is initiating minor compaction (all files) 2024-11-18T07:52:42,946 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/A in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:42,946 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/acea7b8087b44ba280bf0d7ceb86301b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/97fe9bb4056c4891bb382c585de16861, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/1fa21e47b7924dd98d07b4919970c7d3] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=91.4 K 2024-11-18T07:52:42,946 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:42,946 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/acea7b8087b44ba280bf0d7ceb86301b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/97fe9bb4056c4891bb382c585de16861, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/1fa21e47b7924dd98d07b4919970c7d3] 2024-11-18T07:52:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,947 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting c273cad045eb45918c1b852161e30192, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731916359072 2024-11-18T07:52:42,947 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting acea7b8087b44ba280bf0d7ceb86301b, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731916359072 2024-11-18T07:52:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,947 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 53c6f1a50a0e442eb5630fdd60173017, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731916359756 2024-11-18T07:52:42,948 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97fe9bb4056c4891bb382c585de16861, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731916359756 2024-11-18T07:52:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,950 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d38a5786a63441aa0b914d86c802ece, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731916361962 2024-11-18T07:52:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,950 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fa21e47b7924dd98d07b4919970c7d3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731916361962 2024-11-18T07:52:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,968 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:42,971 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#B#compaction#139 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,971 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/d4f6646eb9a44a51af248137556742fe is 50, key is test_row_0/B:col10/1731916362290/Put/seqid=0 2024-11-18T07:52:42,971 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111842bd0420d2434b36a4bffbf7e53d9206_3f3b11bb24dc0fc107b3c002b8506511 store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T07:52:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:42,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T07:52:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:42,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:42,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:42,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:42,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:42,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:42,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:42,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,975 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111842bd0420d2434b36a4bffbf7e53d9206_3f3b11bb24dc0fc107b3c002b8506511, store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:42,975 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111842bd0420d2434b36a4bffbf7e53d9206_3f3b11bb24dc0fc107b3c002b8506511 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:42,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741988_1164 (size=12493) 2024-11-18T07:52:43,003 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/d4f6646eb9a44a51af248137556742fe as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/d4f6646eb9a44a51af248137556742fe 2024-11-18T07:52:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,015 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/B of 3f3b11bb24dc0fc107b3c002b8506511 into d4f6646eb9a44a51af248137556742fe(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:43,015 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:43,015 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/B, priority=13, startTime=1731916362942; duration=0sec 2024-11-18T07:52:43,015 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:43,016 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:B 2024-11-18T07:52:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,016 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,020 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:43,020 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/C is initiating minor compaction (all files) 2024-11-18T07:52:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,020 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/C in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:43,020 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/1ccfc07f1ea9474fa45d4e00341999c6, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/8d103e618f53481c895c071945d50703, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/9af01dde2e2d4e70b41388bcd8901109] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=35.8 K 2024-11-18T07:52:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,022 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ccfc07f1ea9474fa45d4e00341999c6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731916359072 2024-11-18T07:52:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fb154e68d6f748c7be73b09e37646c16_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916362971/Put/seqid=0 2024-11-18T07:52:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,032 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d103e618f53481c895c071945d50703, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731916359756 2024-11-18T07:52:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,033 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9af01dde2e2d4e70b41388bcd8901109, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731916361962 2024-11-18T07:52:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741989_1165 (size=4469) 2024-11-18T07:52:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741990_1166 (size=12304) 2024-11-18T07:52:43,059 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#A#compaction#138 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:43,060 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/a822452e855d4d9daf33b360f2e782fe is 175, key is test_row_0/A:col10/1731916362290/Put/seqid=0 2024-11-18T07:52:43,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,060 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,066 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:43,067 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/d49dd181f6b74529a9136e903c1e317e is 50, key is test_row_0/C:col10/1731916362290/Put/seqid=0 2024-11-18T07:52:43,070 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fb154e68d6f748c7be73b09e37646c16_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fb154e68d6f748c7be73b09e37646c16_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:43,077 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/a58c7455016748789881af024878b4cf, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:43,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/a58c7455016748789881af024878b4cf is 175, key is test_row_0/A:col10/1731916362971/Put/seqid=0 2024-11-18T07:52:43,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39334, totalHeapSize=89576, min timestamp=1731916362975, max timestamp=1731916363069Num uniques 859; 2024-11-18T07:52:43,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=39334, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-18T07:52:43,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39334, totalHeapSize=89576, min timestamp=1731916362975, max timestamp=1731916363069Num uniques 859; 2024-11-18T07:52:43,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=39334, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-18T07:52:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916423073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916423076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:43,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916423095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916423098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741992_1168 (size=12493) 2024-11-18T07:52:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916423098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741991_1167 (size=31447) 2024-11-18T07:52:43,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T07:52:43,134 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/d49dd181f6b74529a9136e903c1e317e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/d49dd181f6b74529a9136e903c1e317e 2024-11-18T07:52:43,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741993_1169 (size=31101) 2024-11-18T07:52:43,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T07:52:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:43,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,155 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/C of 3f3b11bb24dc0fc107b3c002b8506511 into d49dd181f6b74529a9136e903c1e317e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:43,156 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:43,156 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/C, priority=13, startTime=1731916362942; duration=0sec 2024-11-18T07:52:43,156 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:43,156 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:C 2024-11-18T07:52:43,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916423208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916423208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916423209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916423231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916423239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T07:52:43,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:43,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:43,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:43,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916423420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916423423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916423431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T07:52:43,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T07:52:43,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:43,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:43,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:43,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916423448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916423449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,541 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/a58c7455016748789881af024878b4cf 2024-11-18T07:52:43,545 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/a822452e855d4d9daf33b360f2e782fe as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a822452e855d4d9daf33b360f2e782fe 2024-11-18T07:52:43,555 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/A of 3f3b11bb24dc0fc107b3c002b8506511 into a822452e855d4d9daf33b360f2e782fe(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:43,555 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:43,555 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/A, priority=13, startTime=1731916362941; duration=0sec 2024-11-18T07:52:43,555 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:43,555 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:A 2024-11-18T07:52:43,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/e004941d36ab4dd19f6eae3cd7f152be is 50, key is test_row_0/B:col10/1731916362971/Put/seqid=0 2024-11-18T07:52:43,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T07:52:43,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:43,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:43,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:43,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741994_1170 (size=9757) 2024-11-18T07:52:43,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916423738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916423738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916423747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T07:52:43,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:43,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:43,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:43,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916423758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916423759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:43,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T07:52:43,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:43,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:43,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:43,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T07:52:44,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/e004941d36ab4dd19f6eae3cd7f152be 2024-11-18T07:52:44,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/0d0b61a273ba4f0ab3ad1bda925d7bb3 is 50, key is test_row_0/C:col10/1731916362971/Put/seqid=0 2024-11-18T07:52:44,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T07:52:44,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:44,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:44,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:44,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:44,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:44,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741995_1171 (size=9757) 2024-11-18T07:52:44,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/0d0b61a273ba4f0ab3ad1bda925d7bb3 2024-11-18T07:52:44,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/a58c7455016748789881af024878b4cf as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a58c7455016748789881af024878b4cf 2024-11-18T07:52:44,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a58c7455016748789881af024878b4cf, entries=150, sequenceid=194, filesize=30.4 K 2024-11-18T07:52:44,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/e004941d36ab4dd19f6eae3cd7f152be as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/e004941d36ab4dd19f6eae3cd7f152be 2024-11-18T07:52:44,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/e004941d36ab4dd19f6eae3cd7f152be, entries=100, sequenceid=194, filesize=9.5 K 2024-11-18T07:52:44,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/0d0b61a273ba4f0ab3ad1bda925d7bb3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0d0b61a273ba4f0ab3ad1bda925d7bb3 2024-11-18T07:52:44,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0d0b61a273ba4f0ab3ad1bda925d7bb3, entries=100, sequenceid=194, filesize=9.5 K 2024-11-18T07:52:44,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=140.89 KB/144270 for 3f3b11bb24dc0fc107b3c002b8506511 in 1172ms, sequenceid=194, compaction requested=false 2024-11-18T07:52:44,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:44,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T07:52:44,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:44,221 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=140.89 KB heapSize=366.13 KB 2024-11-18T07:52:44,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:44,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:44,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:44,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:44,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:44,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:44,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111843069f46bf314bb7b577a5a5a4875353_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916363082/Put/seqid=0 2024-11-18T07:52:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:44,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:44,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741996_1172 (size=12304) 2024-11-18T07:52:44,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:44,299 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111843069f46bf314bb7b577a5a5a4875353_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111843069f46bf314bb7b577a5a5a4875353_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:44,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/5c556dd4e6f84187aa21900eb9648404, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:44,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/5c556dd4e6f84187aa21900eb9648404 is 175, key is test_row_0/A:col10/1731916363082/Put/seqid=0 2024-11-18T07:52:44,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916424294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916424295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916424287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916424300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916424308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741997_1173 (size=31105) 2024-11-18T07:52:44,321 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/5c556dd4e6f84187aa21900eb9648404 2024-11-18T07:52:44,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/aeea1f07f12b4b1d9e8bec6d8b996496 is 50, key is test_row_0/B:col10/1731916363082/Put/seqid=0 2024-11-18T07:52:44,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741998_1174 (size=12151) 2024-11-18T07:52:44,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916424418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916424418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916424418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916424418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916424418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916424631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916424631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916424631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916424631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916424631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,742 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/aeea1f07f12b4b1d9e8bec6d8b996496 2024-11-18T07:52:44,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/106e1b84527c42059ad1fed92ae208ff is 50, key is test_row_0/C:col10/1731916363082/Put/seqid=0 2024-11-18T07:52:44,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741999_1175 (size=12151) 2024-11-18T07:52:44,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916424948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916424950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916424950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T07:52:44,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916424952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:44,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916424939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:45,231 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/106e1b84527c42059ad1fed92ae208ff 2024-11-18T07:52:45,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/5c556dd4e6f84187aa21900eb9648404 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/5c556dd4e6f84187aa21900eb9648404 2024-11-18T07:52:45,277 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/5c556dd4e6f84187aa21900eb9648404, entries=150, sequenceid=221, filesize=30.4 K 2024-11-18T07:52:45,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/aeea1f07f12b4b1d9e8bec6d8b996496 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/aeea1f07f12b4b1d9e8bec6d8b996496 2024-11-18T07:52:45,290 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/aeea1f07f12b4b1d9e8bec6d8b996496, entries=150, sequenceid=221, filesize=11.9 K 2024-11-18T07:52:45,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/106e1b84527c42059ad1fed92ae208ff as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/106e1b84527c42059ad1fed92ae208ff 2024-11-18T07:52:45,302 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/106e1b84527c42059ad1fed92ae208ff, entries=150, sequenceid=221, filesize=11.9 K 2024-11-18T07:52:45,303 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~366.08 KB/374864, currentSize=60.38 KB/61830 for 3f3b11bb24dc0fc107b3c002b8506511 in 1082ms, sequenceid=221, compaction requested=true 2024-11-18T07:52:45,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:45,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:45,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-18T07:52:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-18T07:52:45,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-18T07:52:45,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4900 sec 2024-11-18T07:52:45,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.5060 sec 2024-11-18T07:52:45,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T07:52:45,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:45,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:45,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:45,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:45,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:45,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:45,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c7ea9577b6a747089dc76e8518f2f2b8_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916364298/Put/seqid=0 2024-11-18T07:52:45,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742000_1176 (size=12304) 2024-11-18T07:52:45,542 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:45,548 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c7ea9577b6a747089dc76e8518f2f2b8_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c7ea9577b6a747089dc76e8518f2f2b8_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:45,549 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/3be051213876435cb019a72697de9880, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:45,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/3be051213876435cb019a72697de9880 is 175, key is test_row_0/A:col10/1731916364298/Put/seqid=0 2024-11-18T07:52:45,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:45,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916425534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916425531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:45,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916425535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:45,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916425550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:45,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916425551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:45,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742001_1177 (size=31105) 2024-11-18T07:52:45,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916425659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:45,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916425659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:45,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916425659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:45,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916425671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:45,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916425673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:45,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:45,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916425878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:45,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:45,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916425879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:45,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916425879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:45,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916425879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:45,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916425899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:45,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:45,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:46,011 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/3be051213876435cb019a72697de9880 2024-11-18T07:52:46,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/8ece6014f84842c7ae29299c4fa667d6 is 50, key is test_row_0/B:col10/1731916364298/Put/seqid=0 2024-11-18T07:52:46,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742002_1178 (size=12151) 2024-11-18T07:52:46,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916426199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:46,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916426210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:46,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916426207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:46,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:46,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916426211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:46,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:46,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916426218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:46,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/8ece6014f84842c7ae29299c4fa667d6 2024-11-18T07:52:46,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/78c25ca19c4645d29e90a40bf4be14d2 is 50, key is test_row_0/C:col10/1731916364298/Put/seqid=0 2024-11-18T07:52:46,544 INFO [master/87fb3d252950:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-18T07:52:46,544 INFO [master/87fb3d252950:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-18T07:52:46,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742003_1179 (size=12151) 2024-11-18T07:52:46,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916426709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:46,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916426732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:46,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:46,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916426732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:46,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:46,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:46,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916426733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:46,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:46,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916426733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:46,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:46,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:46,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T07:52:46,970 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:46,970 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:52:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-18T07:52:46,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/78c25ca19c4645d29e90a40bf4be14d2 2024-11-18T07:52:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T07:52:46,973 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:52:46,974 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:52:46,974 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:52:47,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/3be051213876435cb019a72697de9880 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/3be051213876435cb019a72697de9880 2024-11-18T07:52:47,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/3be051213876435cb019a72697de9880, entries=150, sequenceid=234, filesize=30.4 K 2024-11-18T07:52:47,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/8ece6014f84842c7ae29299c4fa667d6 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/8ece6014f84842c7ae29299c4fa667d6 2024-11-18T07:52:47,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/8ece6014f84842c7ae29299c4fa667d6, entries=150, sequenceid=234, filesize=11.9 K 2024-11-18T07:52:47,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/78c25ca19c4645d29e90a40bf4be14d2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/78c25ca19c4645d29e90a40bf4be14d2 2024-11-18T07:52:47,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/78c25ca19c4645d29e90a40bf4be14d2, entries=150, sequenceid=234, filesize=11.9 K 2024-11-18T07:52:47,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 3f3b11bb24dc0fc107b3c002b8506511 in 1596ms, sequenceid=234, compaction requested=true 2024-11-18T07:52:47,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:47,065 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:47,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:47,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:47,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:47,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:47,065 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:47,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:47,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:47,066 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:47,066 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124758 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:47,066 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/B is initiating minor compaction (all files) 2024-11-18T07:52:47,066 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/A is initiating minor compaction (all files) 2024-11-18T07:52:47,067 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/A in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:47,067 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/B in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:47,067 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a822452e855d4d9daf33b360f2e782fe, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a58c7455016748789881af024878b4cf, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/5c556dd4e6f84187aa21900eb9648404, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/3be051213876435cb019a72697de9880] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=121.8 K 2024-11-18T07:52:47,067 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/d4f6646eb9a44a51af248137556742fe, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/e004941d36ab4dd19f6eae3cd7f152be, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/aeea1f07f12b4b1d9e8bec6d8b996496, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/8ece6014f84842c7ae29299c4fa667d6] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=45.5 K 2024-11-18T07:52:47,067 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:47,067 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a822452e855d4d9daf33b360f2e782fe, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a58c7455016748789881af024878b4cf, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/5c556dd4e6f84187aa21900eb9648404, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/3be051213876435cb019a72697de9880] 2024-11-18T07:52:47,068 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting d4f6646eb9a44a51af248137556742fe, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731916361962 2024-11-18T07:52:47,068 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a822452e855d4d9daf33b360f2e782fe, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731916361962 2024-11-18T07:52:47,069 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting e004941d36ab4dd19f6eae3cd7f152be, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731916362298 2024-11-18T07:52:47,069 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a58c7455016748789881af024878b4cf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731916362298 2024-11-18T07:52:47,069 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting aeea1f07f12b4b1d9e8bec6d8b996496, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731916363067 2024-11-18T07:52:47,069 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c556dd4e6f84187aa21900eb9648404, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731916363067 2024-11-18T07:52:47,069 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ece6014f84842c7ae29299c4fa667d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731916364295 2024-11-18T07:52:47,069 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3be051213876435cb019a72697de9880, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731916364295 2024-11-18T07:52:47,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T07:52:47,095 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:47,109 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#B#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:47,110 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/221e48ae87674239a558545950ce58c5 is 50, key is test_row_0/B:col10/1731916364298/Put/seqid=0 2024-11-18T07:52:47,117 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118b4c39c133d104395955cebb02ab2c5ea_3f3b11bb24dc0fc107b3c002b8506511 store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:47,120 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118b4c39c133d104395955cebb02ab2c5ea_3f3b11bb24dc0fc107b3c002b8506511, store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:47,121 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b4c39c133d104395955cebb02ab2c5ea_3f3b11bb24dc0fc107b3c002b8506511 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:47,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T07:52:47,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:47,129 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T07:52:47,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:47,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:47,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:47,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:47,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:47,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:47,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742004_1180 (size=12629) 2024-11-18T07:52:47,189 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/221e48ae87674239a558545950ce58c5 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/221e48ae87674239a558545950ce58c5 2024-11-18T07:52:47,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e338728c4ca3460e8f6babdf02a2d32e_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916365535/Put/seqid=0 2024-11-18T07:52:47,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742005_1181 (size=4469) 2024-11-18T07:52:47,218 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/B of 3f3b11bb24dc0fc107b3c002b8506511 into 221e48ae87674239a558545950ce58c5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:47,218 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:47,218 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/B, priority=12, startTime=1731916367065; duration=0sec 2024-11-18T07:52:47,219 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:47,219 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:B 2024-11-18T07:52:47,219 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:47,221 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:47,222 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/C is initiating minor compaction (all files) 2024-11-18T07:52:47,222 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/C in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:47,222 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/d49dd181f6b74529a9136e903c1e317e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0d0b61a273ba4f0ab3ad1bda925d7bb3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/106e1b84527c42059ad1fed92ae208ff, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/78c25ca19c4645d29e90a40bf4be14d2] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=45.5 K 2024-11-18T07:52:47,222 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting d49dd181f6b74529a9136e903c1e317e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731916361962 2024-11-18T07:52:47,223 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d0b61a273ba4f0ab3ad1bda925d7bb3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731916362298 2024-11-18T07:52:47,223 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 106e1b84527c42059ad1fed92ae208ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731916363067 2024-11-18T07:52:47,225 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 78c25ca19c4645d29e90a40bf4be14d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731916364295 2024-11-18T07:52:47,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742006_1182 (size=12304) 2024-11-18T07:52:47,254 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#C#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:47,254 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/040494a6877a4b84be6b4e39da4383ba is 50, key is test_row_0/C:col10/1731916364298/Put/seqid=0 2024-11-18T07:52:47,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742007_1183 (size=12629) 2024-11-18T07:52:47,269 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/040494a6877a4b84be6b4e39da4383ba as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/040494a6877a4b84be6b4e39da4383ba 2024-11-18T07:52:47,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T07:52:47,302 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/C of 3f3b11bb24dc0fc107b3c002b8506511 into 040494a6877a4b84be6b4e39da4383ba(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:47,302 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:47,303 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/C, priority=12, startTime=1731916367065; duration=0sec 2024-11-18T07:52:47,303 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:47,303 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:C 2024-11-18T07:52:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T07:52:47,611 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#A#compaction#150 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:47,612 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/ade21564eef3405488bc72a7c6f1941b is 175, key is test_row_0/A:col10/1731916364298/Put/seqid=0 2024-11-18T07:52:47,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742008_1184 (size=31583) 2024-11-18T07:52:47,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:47,667 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e338728c4ca3460e8f6babdf02a2d32e_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e338728c4ca3460e8f6babdf02a2d32e_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:47,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/a5e3c7b07c2b40fdb29ee0f24cc98e16, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:47,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/a5e3c7b07c2b40fdb29ee0f24cc98e16 is 175, key is test_row_0/A:col10/1731916365535/Put/seqid=0 2024-11-18T07:52:47,717 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/ade21564eef3405488bc72a7c6f1941b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/ade21564eef3405488bc72a7c6f1941b 2024-11-18T07:52:47,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:47,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742009_1185 (size=31105) 2024-11-18T07:52:47,760 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/A of 3f3b11bb24dc0fc107b3c002b8506511 into ade21564eef3405488bc72a7c6f1941b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:47,760 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:47,760 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/A, priority=12, startTime=1731916367064; duration=0sec 2024-11-18T07:52:47,760 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:47,760 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:A 2024-11-18T07:52:47,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:47,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916427768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:47,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916427769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916427767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916427771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:47,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916427782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:47,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:47,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916427888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:47,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916427899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:47,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916427899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:47,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:47,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916427899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:47,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916427899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:48,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916428098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:48,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:48,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T07:52:48,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916428110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:48,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:48,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916428111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:48,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916428118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:48,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916428130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:48,140 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/a5e3c7b07c2b40fdb29ee0f24cc98e16 2024-11-18T07:52:48,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/542551d8b8ac47e38eec7d846bd52b3e is 50, key is test_row_0/B:col10/1731916365535/Put/seqid=0 2024-11-18T07:52:48,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742010_1186 (size=12151) 2024-11-18T07:52:48,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916428418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:48,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:48,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916428418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:48,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:48,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916428418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:48,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:48,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916428427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:48,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:48,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916428448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:48,579 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/542551d8b8ac47e38eec7d846bd52b3e 2024-11-18T07:52:48,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/755b8765ac6047acb682e2feab2d7f70 is 50, key is test_row_0/C:col10/1731916365535/Put/seqid=0 2024-11-18T07:52:48,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742011_1187 (size=12151) 2024-11-18T07:52:48,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916428928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:48,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916428929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:48,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916428929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:48,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916428938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:48,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916428959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:49,055 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/755b8765ac6047acb682e2feab2d7f70 2024-11-18T07:52:49,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/a5e3c7b07c2b40fdb29ee0f24cc98e16 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a5e3c7b07c2b40fdb29ee0f24cc98e16 2024-11-18T07:52:49,102 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a5e3c7b07c2b40fdb29ee0f24cc98e16, entries=150, sequenceid=254, filesize=30.4 K 2024-11-18T07:52:49,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/542551d8b8ac47e38eec7d846bd52b3e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/542551d8b8ac47e38eec7d846bd52b3e 2024-11-18T07:52:49,116 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/542551d8b8ac47e38eec7d846bd52b3e, entries=150, sequenceid=254, filesize=11.9 K 2024-11-18T07:52:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T07:52:49,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/755b8765ac6047acb682e2feab2d7f70 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/755b8765ac6047acb682e2feab2d7f70 2024-11-18T07:52:49,135 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/755b8765ac6047acb682e2feab2d7f70, entries=150, sequenceid=254, filesize=11.9 K 2024-11-18T07:52:49,141 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 3f3b11bb24dc0fc107b3c002b8506511 in 2013ms, sequenceid=254, compaction requested=false 2024-11-18T07:52:49,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:49,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:49,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-18T07:52:49,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-18T07:52:49,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-18T07:52:49,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1770 sec 2024-11-18T07:52:49,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.1940 sec 2024-11-18T07:52:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:49,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T07:52:49,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:49,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:49,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:49,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:49,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:49,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:49,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b2ff4149e3d547b2b6a89f9d80f92512_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916367768/Put/seqid=0 2024-11-18T07:52:49,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742012_1188 (size=12454) 2024-11-18T07:52:50,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916430025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916430026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916430032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916430035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916430039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916430150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916430151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916430151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916430151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916430151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916430368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916430368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916430368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916430368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916430368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,382 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:50,388 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b2ff4149e3d547b2b6a89f9d80f92512_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b2ff4149e3d547b2b6a89f9d80f92512_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:50,389 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/e48f828d083046e2b36d6ea1dfe6b43e, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:50,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/e48f828d083046e2b36d6ea1dfe6b43e is 175, key is test_row_0/A:col10/1731916367768/Put/seqid=0 2024-11-18T07:52:50,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742013_1189 (size=31255) 2024-11-18T07:52:50,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916430679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916430679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916430679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916430683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916430684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:50,843 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/e48f828d083046e2b36d6ea1dfe6b43e 2024-11-18T07:52:50,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/f902b414beb14ca4b8ff85a2d517dd2d is 50, key is test_row_0/B:col10/1731916367768/Put/seqid=0 2024-11-18T07:52:50,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742014_1190 (size=12301) 2024-11-18T07:52:50,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/f902b414beb14ca4b8ff85a2d517dd2d 2024-11-18T07:52:50,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/b23e9a883b194680b7bc90b8c140aca7 is 50, key is test_row_0/C:col10/1731916367768/Put/seqid=0 2024-11-18T07:52:51,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742015_1191 (size=12301) 2024-11-18T07:52:51,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/b23e9a883b194680b7bc90b8c140aca7 2024-11-18T07:52:51,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/e48f828d083046e2b36d6ea1dfe6b43e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e48f828d083046e2b36d6ea1dfe6b43e 2024-11-18T07:52:51,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e48f828d083046e2b36d6ea1dfe6b43e, entries=150, sequenceid=270, filesize=30.5 K 2024-11-18T07:52:51,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/f902b414beb14ca4b8ff85a2d517dd2d as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/f902b414beb14ca4b8ff85a2d517dd2d 2024-11-18T07:52:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T07:52:51,139 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:51,140 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:52:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-18T07:52:51,142 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:52:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T07:52:51,143 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:52:51,143 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:52:51,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/f902b414beb14ca4b8ff85a2d517dd2d, entries=150, sequenceid=270, filesize=12.0 K 2024-11-18T07:52:51,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/b23e9a883b194680b7bc90b8c140aca7 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/b23e9a883b194680b7bc90b8c140aca7 2024-11-18T07:52:51,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/b23e9a883b194680b7bc90b8c140aca7, entries=150, sequenceid=270, filesize=12.0 K 2024-11-18T07:52:51,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 3f3b11bb24dc0fc107b3c002b8506511 in 1250ms, sequenceid=270, compaction requested=true 2024-11-18T07:52:51,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:51,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:51,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:51,198 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:51,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:51,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:51,198 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:51,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:51,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:51,208 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:51,209 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93943 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:51,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T07:52:51,214 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/A is initiating minor compaction (all files) 2024-11-18T07:52:51,214 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/B is initiating minor compaction (all files) 2024-11-18T07:52:51,215 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/A in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:51,215 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/B in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:51,215 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/ade21564eef3405488bc72a7c6f1941b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a5e3c7b07c2b40fdb29ee0f24cc98e16, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e48f828d083046e2b36d6ea1dfe6b43e] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=91.7 K 2024-11-18T07:52:51,215 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/221e48ae87674239a558545950ce58c5, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/542551d8b8ac47e38eec7d846bd52b3e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/f902b414beb14ca4b8ff85a2d517dd2d] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=36.2 K 2024-11-18T07:52:51,215 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:51,215 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/ade21564eef3405488bc72a7c6f1941b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a5e3c7b07c2b40fdb29ee0f24cc98e16, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e48f828d083046e2b36d6ea1dfe6b43e] 2024-11-18T07:52:51,218 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ade21564eef3405488bc72a7c6f1941b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731916364295 2024-11-18T07:52:51,218 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 221e48ae87674239a558545950ce58c5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731916364295 2024-11-18T07:52:51,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916370032, max timestamp=1731916371204Num uniques 859; 2024-11-18T07:52:51,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:52:51,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916370032, max timestamp=1731916371204Num uniques 859; 2024-11-18T07:52:51,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:52:51,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916370032, max timestamp=1731916371204Num uniques 859; 2024-11-18T07:52:51,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:52:51,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:51,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:51,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:51,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:51,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:51,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:51,222 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5e3c7b07c2b40fdb29ee0f24cc98e16, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731916365529 2024-11-18T07:52:51,223 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 542551d8b8ac47e38eec7d846bd52b3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731916365529 2024-11-18T07:52:51,224 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e48f828d083046e2b36d6ea1dfe6b43e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731916367768 2024-11-18T07:52:51,225 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting f902b414beb14ca4b8ff85a2d517dd2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731916367768 2024-11-18T07:52:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T07:52:51,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184b38513660074890ae589890991dd4a3_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916371204/Put/seqid=0 2024-11-18T07:52:51,271 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#B#compaction#160 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:51,272 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/4d1b1d127ba9496f927ff518b66ae0e3 is 50, key is test_row_0/B:col10/1731916367768/Put/seqid=0 2024-11-18T07:52:51,283 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:51,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T07:52:51,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:51,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:51,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:51,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:51,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:51,328 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411180b08ad2eda57489197f44621be1b63fe_3f3b11bb24dc0fc107b3c002b8506511 store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:51,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916371300, max timestamp=1731916371324Num uniques 859; 2024-11-18T07:52:51,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:52:51,330 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411180b08ad2eda57489197f44621be1b63fe_3f3b11bb24dc0fc107b3c002b8506511, store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:51,331 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180b08ad2eda57489197f44621be1b63fe_3f3b11bb24dc0fc107b3c002b8506511 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:51,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916371300, max timestamp=1731916371324Num uniques 859; 2024-11-18T07:52:51,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:52:51,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916371300, max timestamp=1731916371324Num uniques 859; 2024-11-18T07:52:51,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:52:51,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742016_1192 (size=17534) 2024-11-18T07:52:51,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742017_1193 (size=12881) 2024-11-18T07:52:51,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742018_1194 (size=4469) 2024-11-18T07:52:51,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731916371370, max timestamp=1731916371386Num uniques 1018; 2024-11-18T07:52:51,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T07:52:51,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731916371370, max timestamp=1731916371399Num uniques 1018; 2024-11-18T07:52:51,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T07:52:51,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731916371370, max timestamp=1731916371399Num uniques 1018; 2024-11-18T07:52:51,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T07:52:51,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731916371423, max timestamp=1731916371444Num uniques 1027; 2024-11-18T07:52:51,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=17262, suffix heap size=134776,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=377 2024-11-18T07:52:51,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731916371423, max timestamp=1731916371444Num uniques 1027; 2024-11-18T07:52:51,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=17262, suffix heap size=134776,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=377 2024-11-18T07:52:51,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731916371423, max timestamp=1731916371444Num uniques 1027; 2024-11-18T07:52:51,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=17262, suffix heap size=134776,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=377 2024-11-18T07:52:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T07:52:51,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T07:52:51,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:51,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:51,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:51,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1731916371454, max timestamp=1731916371521Num uniques 1236; 2024-11-18T07:52:51,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1731916371454, max timestamp=1731916371521Num uniques 1236; 2024-11-18T07:52:51,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=17676, suffix heap size=156512,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=386 2024-11-18T07:52:51,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=17676, suffix heap size=156512,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=386 2024-11-18T07:52:51,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1731916371454, max timestamp=1731916371521Num uniques 1236; 2024-11-18T07:52:51,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=17676, suffix heap size=156512,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=386 2024-11-18T07:52:51,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916431599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:51,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916431605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:51,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:51,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916431607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:51,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916431608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:51,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:51,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916431610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:51,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T07:52:51,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:51,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:51,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:51,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:51,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:51,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:51,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916431731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:51,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916431731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:51,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916431731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:51,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916431731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:51,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:51,758 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:51,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916431743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T07:52:51,790 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184b38513660074890ae589890991dd4a3_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184b38513660074890ae589890991dd4a3_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:51,799 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/191da0f1a0774b2994f0d2e34929c3da, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:51,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/191da0f1a0774b2994f0d2e34929c3da is 175, key is test_row_0/A:col10/1731916371204/Put/seqid=0 2024-11-18T07:52:51,803 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/4d1b1d127ba9496f927ff518b66ae0e3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/4d1b1d127ba9496f927ff518b66ae0e3 2024-11-18T07:52:51,804 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#A#compaction#161 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:51,804 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/dd5a4ab7df044a759e7b22e9eb57bd6f is 175, key is test_row_0/A:col10/1731916367768/Put/seqid=0 2024-11-18T07:52:51,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T07:52:51,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:51,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:51,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:51,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:51,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:51,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742019_1195 (size=48639) 2024-11-18T07:52:51,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742020_1196 (size=31835) 2024-11-18T07:52:51,899 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/191da0f1a0774b2994f0d2e34929c3da 2024-11-18T07:52:51,905 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/B of 3f3b11bb24dc0fc107b3c002b8506511 into 4d1b1d127ba9496f927ff518b66ae0e3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:51,905 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:51,905 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/B, priority=13, startTime=1731916371198; duration=0sec 2024-11-18T07:52:51,906 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:51,906 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:B 2024-11-18T07:52:51,906 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:51,926 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:51,927 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/C is initiating minor compaction (all files) 2024-11-18T07:52:51,927 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/C in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:51,927 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/040494a6877a4b84be6b4e39da4383ba, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/755b8765ac6047acb682e2feab2d7f70, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/b23e9a883b194680b7bc90b8c140aca7] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=36.2 K 2024-11-18T07:52:51,931 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 040494a6877a4b84be6b4e39da4383ba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731916364295 2024-11-18T07:52:51,935 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 755b8765ac6047acb682e2feab2d7f70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731916365529 2024-11-18T07:52:51,943 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting b23e9a883b194680b7bc90b8c140aca7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731916367768 2024-11-18T07:52:51,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916431948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:51,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916431948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:51,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916431950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:51,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916431969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:51,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916431969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:51,984 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#C#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:51,985 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/78e18ff1e2d64164a92a7d77626b37f4 is 50, key is test_row_0/C:col10/1731916367768/Put/seqid=0 2024-11-18T07:52:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:52,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/7047155a6d0445adbe45fd590bd4d638 is 50, key is test_row_0/B:col10/1731916371204/Put/seqid=0 2024-11-18T07:52:52,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T07:52:52,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:52,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:52,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742022_1198 (size=12301) 2024-11-18T07:52:52,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742021_1197 (size=12881) 2024-11-18T07:52:52,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T07:52:52,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:52,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:52,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:52,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:52,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:52,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916432266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:52,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:52,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916432271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916432271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T07:52:52,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916432288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:52,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916432288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:52,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:52,331 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/dd5a4ab7df044a759e7b22e9eb57bd6f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/dd5a4ab7df044a759e7b22e9eb57bd6f 2024-11-18T07:52:52,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T07:52:52,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:52,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:52,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:52,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:52,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:52,380 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/A of 3f3b11bb24dc0fc107b3c002b8506511 into dd5a4ab7df044a759e7b22e9eb57bd6f(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-18T07:52:52,380 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:52,381 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/A, priority=13, startTime=1731916371198; duration=1sec 2024-11-18T07:52:52,381 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:52,381 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:A 2024-11-18T07:52:52,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/7047155a6d0445adbe45fd590bd4d638 2024-11-18T07:52:52,485 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/78e18ff1e2d64164a92a7d77626b37f4 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/78e18ff1e2d64164a92a7d77626b37f4 2024-11-18T07:52:52,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/c810b12e0a274997a2e445d792e1e4da is 50, key is test_row_0/C:col10/1731916371204/Put/seqid=0 2024-11-18T07:52:52,512 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/C of 3f3b11bb24dc0fc107b3c002b8506511 into 78e18ff1e2d64164a92a7d77626b37f4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:52,512 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:52,513 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/C, priority=13, startTime=1731916371198; duration=0sec 2024-11-18T07:52:52,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T07:52:52,513 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:52,513 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:C 2024-11-18T07:52:52,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:52,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:52,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:52,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:52,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:52,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742023_1199 (size=12301) 2024-11-18T07:52:52,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/c810b12e0a274997a2e445d792e1e4da 2024-11-18T07:52:52,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/191da0f1a0774b2994f0d2e34929c3da as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/191da0f1a0774b2994f0d2e34929c3da 2024-11-18T07:52:52,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/191da0f1a0774b2994f0d2e34929c3da, entries=250, sequenceid=292, filesize=47.5 K 2024-11-18T07:52:52,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/7047155a6d0445adbe45fd590bd4d638 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/7047155a6d0445adbe45fd590bd4d638 2024-11-18T07:52:52,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/7047155a6d0445adbe45fd590bd4d638, entries=150, sequenceid=292, filesize=12.0 K 2024-11-18T07:52:52,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/c810b12e0a274997a2e445d792e1e4da as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/c810b12e0a274997a2e445d792e1e4da 2024-11-18T07:52:52,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/c810b12e0a274997a2e445d792e1e4da, entries=150, sequenceid=292, filesize=12.0 K 2024-11-18T07:52:52,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=161.02 KB/164880 for 3f3b11bb24dc0fc107b3c002b8506511 in 1429ms, sequenceid=292, compaction requested=false 2024-11-18T07:52:52,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:52,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T07:52:52,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:52,684 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=161.02 KB heapSize=443.39 KB 2024-11-18T07:52:52,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:52,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:52,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:52,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:52,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:52,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:52,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d123181b064c42069e08a9d7d9588c92_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916371596/Put/seqid=0 2024-11-18T07:52:52,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742024_1200 (size=12454) 2024-11-18T07:52:52,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:52,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916432812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:52,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916432812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:52,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916432815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:52,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916432823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:52,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:52,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916432829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:52,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916432930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:52,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916432938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:52,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916432940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:53,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916433148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:53,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916433149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:53,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916433151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:53,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:53,174 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d123181b064c42069e08a9d7d9588c92_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d123181b064c42069e08a9d7d9588c92_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:53,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/e5b77fef5c9341e2a4d4333372f56a69, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:53,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/e5b77fef5c9341e2a4d4333372f56a69 is 175, key is test_row_0/A:col10/1731916371596/Put/seqid=0 2024-11-18T07:52:53,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742025_1201 (size=31255) 2024-11-18T07:52:53,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T07:52:53,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916433458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916433459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:53,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916433479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:53,607 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/e5b77fef5c9341e2a4d4333372f56a69 2024-11-18T07:52:53,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/1102ddf2cffd45afa05c9144d414c5b1 is 50, key is test_row_0/B:col10/1731916371596/Put/seqid=0 2024-11-18T07:52:53,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742026_1202 (size=12301) 2024-11-18T07:52:53,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916433828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:53,837 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_0' in 3f3b11bb24dc0fc107b3c002b8506511 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:52:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:53,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916433848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:53,854 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_1' in 3f3b11bb24dc0fc107b3c002b8506511 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:52:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:53,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916433968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:53,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916433968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:53,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916433988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:54,103 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/1102ddf2cffd45afa05c9144d414c5b1 2024-11-18T07:52:54,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/366ab0a10db54791a253befc8636ea50 is 50, key is test_row_0/C:col10/1731916371596/Put/seqid=0 2024-11-18T07:52:54,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742027_1203 (size=12301) 2024-11-18T07:52:54,535 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/366ab0a10db54791a253befc8636ea50 2024-11-18T07:52:54,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/e5b77fef5c9341e2a4d4333372f56a69 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e5b77fef5c9341e2a4d4333372f56a69 2024-11-18T07:52:54,556 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e5b77fef5c9341e2a4d4333372f56a69, entries=150, sequenceid=383, filesize=30.5 K 2024-11-18T07:52:54,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/1102ddf2cffd45afa05c9144d414c5b1 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/1102ddf2cffd45afa05c9144d414c5b1 2024-11-18T07:52:54,591 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/1102ddf2cffd45afa05c9144d414c5b1, entries=150, sequenceid=383, filesize=12.0 K 2024-11-18T07:52:54,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/366ab0a10db54791a253befc8636ea50 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/366ab0a10db54791a253befc8636ea50 2024-11-18T07:52:54,605 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/366ab0a10db54791a253befc8636ea50, entries=150, sequenceid=383, filesize=12.0 K 2024-11-18T07:52:54,610 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~443.34 KB/453984, currentSize=33.54 KB/34350 for 3f3b11bb24dc0fc107b3c002b8506511 in 1922ms, sequenceid=383, compaction requested=true 2024-11-18T07:52:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-18T07:52:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-18T07:52:54,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-18T07:52:54,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4690 sec 2024-11-18T07:52:54,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 3.4770 sec 2024-11-18T07:52:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:54,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:52:54,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:54,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:54,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:54,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:54,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:54,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:54,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b8228bbe55374535b41f09dd2e8e0fb0_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916372810/Put/seqid=0 2024-11-18T07:52:55,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742028_1204 (size=14994) 2024-11-18T07:52:55,009 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:55,015 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b8228bbe55374535b41f09dd2e8e0fb0_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b8228bbe55374535b41f09dd2e8e0fb0_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:55,016 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/7616cd87bf3a4d8fb4f3f5e7fc027c96, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:55,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/7616cd87bf3a4d8fb4f3f5e7fc027c96 is 175, key is test_row_0/A:col10/1731916372810/Put/seqid=0 2024-11-18T07:52:55,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742029_1205 (size=39949) 2024-11-18T07:52:55,033 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/7616cd87bf3a4d8fb4f3f5e7fc027c96 2024-11-18T07:52:55,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/e574baf85d904acda6694dc4d0b62e9a is 50, key is test_row_0/B:col10/1731916372810/Put/seqid=0 2024-11-18T07:52:55,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731916375037, max timestamp=1731916375057Num uniques 859; 2024-11-18T07:52:55,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:52:55,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731916375037, max timestamp=1731916375057Num uniques 859; 2024-11-18T07:52:55,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:52:55,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916375041, max timestamp=1731916375057Num uniques 859; 2024-11-18T07:52:55,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:52:55,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742030_1206 (size=12301) 2024-11-18T07:52:55,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916435087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:55,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916435094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:55,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916435095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:55,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916435198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:55,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916435211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:55,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:55,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916435211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:55,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T07:52:55,319 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:55,320 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:52:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-18T07:52:55,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T07:52:55,329 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:52:55,330 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:52:55,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:52:55,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916435410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:55,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916435419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T07:52:55,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916435430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:55,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:55,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/e574baf85d904acda6694dc4d0b62e9a 2024-11-18T07:52:55,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T07:52:55,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:55,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:55,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:55,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:55,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:55,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/392ed8b0b31946a9a984d2486d08aeed is 50, key is test_row_0/C:col10/1731916372810/Put/seqid=0 2024-11-18T07:52:55,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742031_1207 (size=12301) 2024-11-18T07:52:55,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T07:52:55,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:55,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:55,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:55,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:55,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T07:52:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:55,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916435728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:55,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916435738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:55,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916435751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:55,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T07:52:55,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:55,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:55,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:55,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:55,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:55,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48824 deadline: 1731916435862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:55,871 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_0' in 3f3b11bb24dc0fc107b3c002b8506511 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:52:55,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:55,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48816 deadline: 1731916435879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:55,887 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_1' in 3f3b11bb24dc0fc107b3c002b8506511 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4284 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:52:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:55,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/392ed8b0b31946a9a984d2486d08aeed 2024-11-18T07:52:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T07:52:55,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T07:52:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:55,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:55,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/7616cd87bf3a4d8fb4f3f5e7fc027c96 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/7616cd87bf3a4d8fb4f3f5e7fc027c96 2024-11-18T07:52:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:52:56,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/7616cd87bf3a4d8fb4f3f5e7fc027c96, entries=200, sequenceid=393, filesize=39.0 K 2024-11-18T07:52:56,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/e574baf85d904acda6694dc4d0b62e9a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/e574baf85d904acda6694dc4d0b62e9a 2024-11-18T07:52:56,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/e574baf85d904acda6694dc4d0b62e9a, entries=150, sequenceid=393, filesize=12.0 K 2024-11-18T07:52:56,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/392ed8b0b31946a9a984d2486d08aeed as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/392ed8b0b31946a9a984d2486d08aeed 2024-11-18T07:52:56,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/392ed8b0b31946a9a984d2486d08aeed, entries=150, sequenceid=393, filesize=12.0 K 2024-11-18T07:52:56,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.65 KB/141980 for 3f3b11bb24dc0fc107b3c002b8506511 in 1095ms, sequenceid=393, compaction requested=true 2024-11-18T07:52:56,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:56,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:56,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:56,085 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:56,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:56,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:56,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:56,086 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:56,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:56,099 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151678 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:56,099 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/A is initiating minor compaction (all files) 2024-11-18T07:52:56,099 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/A in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:56,099 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/dd5a4ab7df044a759e7b22e9eb57bd6f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/191da0f1a0774b2994f0d2e34929c3da, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e5b77fef5c9341e2a4d4333372f56a69, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/7616cd87bf3a4d8fb4f3f5e7fc027c96] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=148.1 K 2024-11-18T07:52:56,099 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:56,100 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/dd5a4ab7df044a759e7b22e9eb57bd6f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/191da0f1a0774b2994f0d2e34929c3da, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e5b77fef5c9341e2a4d4333372f56a69, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/7616cd87bf3a4d8fb4f3f5e7fc027c96] 2024-11-18T07:52:56,100 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:56,100 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/B is initiating minor compaction (all files) 2024-11-18T07:52:56,100 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/B in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:56,100 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/4d1b1d127ba9496f927ff518b66ae0e3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/7047155a6d0445adbe45fd590bd4d638, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/1102ddf2cffd45afa05c9144d414c5b1, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/e574baf85d904acda6694dc4d0b62e9a] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=48.6 K 2024-11-18T07:52:56,103 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting dd5a4ab7df044a759e7b22e9eb57bd6f, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731916367768 2024-11-18T07:52:56,103 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d1b1d127ba9496f927ff518b66ae0e3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731916367768 2024-11-18T07:52:56,107 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 191da0f1a0774b2994f0d2e34929c3da, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731916370032 2024-11-18T07:52:56,107 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7047155a6d0445adbe45fd590bd4d638, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731916370032 2024-11-18T07:52:56,111 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting e5b77fef5c9341e2a4d4333372f56a69, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731916371582 2024-11-18T07:52:56,111 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1102ddf2cffd45afa05c9144d414c5b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731916371582 2024-11-18T07:52:56,115 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7616cd87bf3a4d8fb4f3f5e7fc027c96, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731916372799 2024-11-18T07:52:56,115 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e574baf85d904acda6694dc4d0b62e9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731916372810 2024-11-18T07:52:56,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T07:52:56,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:56,152 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=138.65 KB heapSize=390.45 KB 2024-11-18T07:52:56,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:56,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:56,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:56,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:56,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:56,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:52:56,176 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:56,181 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#B#compaction#172 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:56,182 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/6304b2be87d840d497df340435a6ce69 is 50, key is test_row_0/B:col10/1731916372810/Put/seqid=0 2024-11-18T07:52:56,201 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118ce6430015e75443b969837c54e5644f1_3f3b11bb24dc0fc107b3c002b8506511 store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:56,204 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118ce6430015e75443b969837c54e5644f1_3f3b11bb24dc0fc107b3c002b8506511, store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:56,204 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ce6430015e75443b969837c54e5644f1_3f3b11bb24dc0fc107b3c002b8506511 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:56,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b1786aa2cccd4b92a391e0951dc481f8_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916375093/Put/seqid=0 2024-11-18T07:52:56,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742032_1208 (size=13017) 2024-11-18T07:52:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:56,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. as already flushing 2024-11-18T07:52:56,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742033_1209 (size=4469) 2024-11-18T07:52:56,279 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#A#compaction#171 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:56,280 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/d953fa8ac74e4b33aa163695b148bc57 is 175, key is test_row_0/A:col10/1731916372810/Put/seqid=0 2024-11-18T07:52:56,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742034_1210 (size=12454) 2024-11-18T07:52:56,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:56,308 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b1786aa2cccd4b92a391e0951dc481f8_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b1786aa2cccd4b92a391e0951dc481f8_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:56,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916436299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:56,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916436307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:56,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/f683050905d4454990b17fd903be46e0, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916436315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:56,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/f683050905d4454990b17fd903be46e0 is 175, key is test_row_0/A:col10/1731916375093/Put/seqid=0 2024-11-18T07:52:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:56,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742035_1211 (size=31971) 2024-11-18T07:52:56,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742036_1212 (size=31255) 2024-11-18T07:52:56,388 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/d953fa8ac74e4b33aa163695b148bc57 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/d953fa8ac74e4b33aa163695b148bc57 2024-11-18T07:52:56,394 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/A of 3f3b11bb24dc0fc107b3c002b8506511 into d953fa8ac74e4b33aa163695b148bc57(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:56,394 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:56,394 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/A, priority=12, startTime=1731916376085; duration=0sec 2024-11-18T07:52:56,395 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:56,395 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:A 2024-11-18T07:52:56,395 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:52:56,397 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:52:56,397 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/C is initiating minor compaction (all files) 2024-11-18T07:52:56,397 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/C in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:56,397 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/78e18ff1e2d64164a92a7d77626b37f4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/c810b12e0a274997a2e445d792e1e4da, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/366ab0a10db54791a253befc8636ea50, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/392ed8b0b31946a9a984d2486d08aeed] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=48.6 K 2024-11-18T07:52:56,399 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 78e18ff1e2d64164a92a7d77626b37f4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731916367768 2024-11-18T07:52:56,399 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting c810b12e0a274997a2e445d792e1e4da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731916370032 2024-11-18T07:52:56,407 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 366ab0a10db54791a253befc8636ea50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731916371582 2024-11-18T07:52:56,413 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 392ed8b0b31946a9a984d2486d08aeed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731916372810 2024-11-18T07:52:56,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916436418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:56,438 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:56,438 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/0321a56ef452499bbf49e06b2e27f6d2 is 50, key is test_row_0/C:col10/1731916372810/Put/seqid=0 2024-11-18T07:52:56,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916436439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:56,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916436443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:56,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T07:52:56,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742037_1213 (size=13017) 2024-11-18T07:52:56,503 INFO [Thread-632 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-632. 2024-11-18T07:52:56,503 DEBUG [Thread-632 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:56,503 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:56,503 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:56,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:56,504 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/0321a56ef452499bbf49e06b2e27f6d2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0321a56ef452499bbf49e06b2e27f6d2 2024-11-18T07:52:56,513 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-11-18T07:52:56,513 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:56,513 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:56,513 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:56,513 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:56,513 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-11-18T07:52:56,513 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:56,513 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:56,513 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:56,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:56,516 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/C of 3f3b11bb24dc0fc107b3c002b8506511 into 0321a56ef452499bbf49e06b2e27f6d2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:56,516 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:56,516 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/C, priority=12, startTime=1731916376085; duration=0sec 2024-11-18T07:52:56,516 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:56,516 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:C 2024-11-18T07:52:56,519 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-11-18T07:52:56,519 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:56,519 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:56,519 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:56,522 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:56,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916436638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:56,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:56,647 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/6304b2be87d840d497df340435a6ce69 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/6304b2be87d840d497df340435a6ce69 2024-11-18T07:52:56,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916436649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:56,659 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/B of 3f3b11bb24dc0fc107b3c002b8506511 into 6304b2be87d840d497df340435a6ce69(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:56,659 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:56,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:56,659 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/B, priority=12, startTime=1731916376085; duration=0sec 2024-11-18T07:52:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916436658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:56,659 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:56,659 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:B 2024-11-18T07:52:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:56,775 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=429, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/f683050905d4454990b17fd903be46e0 2024-11-18T07:52:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/a412436f49254b40bce53b0a99639f97 is 50, key is test_row_0/B:col10/1731916375093/Put/seqid=0 2024-11-18T07:52:56,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742038_1214 (size=12301) 2024-11-18T07:52:56,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916436948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:56,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916436958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:56,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916436968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:56,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:56,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:57,191 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/a412436f49254b40bce53b0a99639f97 2024-11-18T07:52:57,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/fc2357d6a9b74a2d806f20633686e9df is 50, key is test_row_0/C:col10/1731916375093/Put/seqid=0 2024-11-18T07:52:57,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742039_1215 (size=12301) 2024-11-18T07:52:57,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1731916437458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:57,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48838 deadline: 1731916437468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:57,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:57,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:57,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:57,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:52:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48870 deadline: 1731916437479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 2024-11-18T07:52:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:52:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:52:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T07:52:57,609 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/fc2357d6a9b74a2d806f20633686e9df 2024-11-18T07:52:57,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/f683050905d4454990b17fd903be46e0 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/f683050905d4454990b17fd903be46e0 2024-11-18T07:52:57,622 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/f683050905d4454990b17fd903be46e0, entries=150, sequenceid=429, filesize=30.5 K 2024-11-18T07:52:57,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/a412436f49254b40bce53b0a99639f97 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/a412436f49254b40bce53b0a99639f97 2024-11-18T07:52:57,634 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/a412436f49254b40bce53b0a99639f97, entries=150, sequenceid=429, filesize=12.0 K 2024-11-18T07:52:57,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/fc2357d6a9b74a2d806f20633686e9df as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/fc2357d6a9b74a2d806f20633686e9df 2024-11-18T07:52:57,644 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/fc2357d6a9b74a2d806f20633686e9df, entries=150, sequenceid=429, filesize=12.0 K 2024-11-18T07:52:57,645 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~390.41 KB/399776, currentSize=46.96 KB/48090 for 3f3b11bb24dc0fc107b3c002b8506511 in 1493ms, sequenceid=429, compaction requested=false 2024-11-18T07:52:57,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:57,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:57,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-18T07:52:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-18T07:52:57,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-18T07:52:57,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3170 sec 2024-11-18T07:52:57,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.3310 sec 2024-11-18T07:52:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:58,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T07:52:58,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:52:58,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:58,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:52:58,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:58,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:52:58,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:52:58,472 INFO [Thread-621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-621. 2024-11-18T07:52:58,472 DEBUG [Thread-621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:58,472 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:58,472 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:58,472 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:58,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111898c0aa6d615a457d9552c598c36d3ad6_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916376306/Put/seqid=0 2024-11-18T07:52:58,481 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-11-18T07:52:58,481 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:58,481 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:58,481 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:58,482 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:58,492 INFO [Thread-627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-627. 2024-11-18T07:52:58,492 DEBUG [Thread-627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:58,492 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:58,493 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:58,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:58,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742040_1216 (size=12454) 2024-11-18T07:52:58,495 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:52:58,506 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111898c0aa6d615a457d9552c598c36d3ad6_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111898c0aa6d615a457d9552c598c36d3ad6_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:52:58,514 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/75b98e12be784401a8f3fd55bdbbe706, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:58,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/75b98e12be784401a8f3fd55bdbbe706 is 175, key is test_row_0/A:col10/1731916376306/Put/seqid=0 2024-11-18T07:52:58,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742041_1217 (size=31255) 2024-11-18T07:52:58,924 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=443, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/75b98e12be784401a8f3fd55bdbbe706 2024-11-18T07:52:58,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/0a8a0cc135f241deaa8641808da8dc51 is 50, key is test_row_0/B:col10/1731916376306/Put/seqid=0 2024-11-18T07:52:58,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742042_1218 (size=12301) 2024-11-18T07:52:59,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/0a8a0cc135f241deaa8641808da8dc51 2024-11-18T07:52:59,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/23ce947f014945249b65c6df15e36711 is 50, key is test_row_0/C:col10/1731916376306/Put/seqid=0 2024-11-18T07:52:59,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742043_1219 (size=12301) 2024-11-18T07:52:59,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/23ce947f014945249b65c6df15e36711 2024-11-18T07:52:59,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/75b98e12be784401a8f3fd55bdbbe706 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/75b98e12be784401a8f3fd55bdbbe706 2024-11-18T07:52:59,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/75b98e12be784401a8f3fd55bdbbe706, entries=150, sequenceid=443, filesize=30.5 K 2024-11-18T07:52:59,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/0a8a0cc135f241deaa8641808da8dc51 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/0a8a0cc135f241deaa8641808da8dc51 2024-11-18T07:52:59,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/0a8a0cc135f241deaa8641808da8dc51, entries=150, sequenceid=443, filesize=12.0 K 2024-11-18T07:52:59,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/23ce947f014945249b65c6df15e36711 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/23ce947f014945249b65c6df15e36711 2024-11-18T07:52:59,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/23ce947f014945249b65c6df15e36711, entries=150, sequenceid=443, filesize=12.0 K 2024-11-18T07:52:59,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=13.42 KB/13740 for 3f3b11bb24dc0fc107b3c002b8506511 in 1045ms, sequenceid=443, compaction requested=true 2024-11-18T07:52:59,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:59,515 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:59,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:52:59,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:52:59,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:52:59,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:59,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f3b11bb24dc0fc107b3c002b8506511:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:52:59,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:52:59,517 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:59,517 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/A is initiating minor compaction (all files) 2024-11-18T07:52:59,517 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/A in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:59,518 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/d953fa8ac74e4b33aa163695b148bc57, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/f683050905d4454990b17fd903be46e0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/75b98e12be784401a8f3fd55bdbbe706] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=92.3 K 2024-11-18T07:52:59,518 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:59,518 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/d953fa8ac74e4b33aa163695b148bc57, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/f683050905d4454990b17fd903be46e0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/75b98e12be784401a8f3fd55bdbbe706] 2024-11-18T07:52:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T07:52:59,518 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:52:59,518 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:59,519 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting d953fa8ac74e4b33aa163695b148bc57, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731916372810 2024-11-18T07:52:59,521 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting f683050905d4454990b17fd903be46e0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731916375079 2024-11-18T07:52:59,521 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 75b98e12be784401a8f3fd55bdbbe706, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731916376275 2024-11-18T07:52:59,524 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:59,524 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/B is initiating minor compaction (all files) 2024-11-18T07:52:59,524 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/B in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:59,525 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/6304b2be87d840d497df340435a6ce69, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/a412436f49254b40bce53b0a99639f97, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/0a8a0cc135f241deaa8641808da8dc51] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=36.7 K 2024-11-18T07:52:59,525 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6304b2be87d840d497df340435a6ce69, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731916372810 2024-11-18T07:52:59,526 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a412436f49254b40bce53b0a99639f97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731916375079 2024-11-18T07:52:59,527 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a8a0cc135f241deaa8641808da8dc51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731916376275 2024-11-18T07:52:59,544 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:59,549 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118619e41d21e1548929e592e544223472f_3f3b11bb24dc0fc107b3c002b8506511 store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:59,589 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#B#compaction#181 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:59,590 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/d4dc402086c0438f8512857c6dc673a3 is 50, key is test_row_0/B:col10/1731916376306/Put/seqid=0 2024-11-18T07:52:59,605 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118619e41d21e1548929e592e544223472f_3f3b11bb24dc0fc107b3c002b8506511, store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:59,606 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118619e41d21e1548929e592e544223472f_3f3b11bb24dc0fc107b3c002b8506511 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:52:59,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742044_1220 (size=13119) 2024-11-18T07:52:59,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742045_1221 (size=4469) 2024-11-18T07:52:59,679 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#A#compaction#180 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:59,680 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/e99b27296ac447bd92f19b252a05a347 is 175, key is test_row_0/A:col10/1731916376306/Put/seqid=0 2024-11-18T07:52:59,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742046_1222 (size=32073) 2024-11-18T07:52:59,774 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/e99b27296ac447bd92f19b252a05a347 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e99b27296ac447bd92f19b252a05a347 2024-11-18T07:52:59,804 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/A of 3f3b11bb24dc0fc107b3c002b8506511 into e99b27296ac447bd92f19b252a05a347(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:52:59,804 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:52:59,804 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/A, priority=13, startTime=1731916379515; duration=0sec 2024-11-18T07:52:59,804 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:52:59,804 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:A 2024-11-18T07:52:59,804 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:52:59,831 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:52:59,831 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 3f3b11bb24dc0fc107b3c002b8506511/C is initiating minor compaction (all files) 2024-11-18T07:52:59,831 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f3b11bb24dc0fc107b3c002b8506511/C in TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:52:59,831 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0321a56ef452499bbf49e06b2e27f6d2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/fc2357d6a9b74a2d806f20633686e9df, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/23ce947f014945249b65c6df15e36711] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp, totalSize=36.7 K 2024-11-18T07:52:59,839 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0321a56ef452499bbf49e06b2e27f6d2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731916372810 2024-11-18T07:52:59,843 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting fc2357d6a9b74a2d806f20633686e9df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731916375079 2024-11-18T07:52:59,853 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 23ce947f014945249b65c6df15e36711, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731916376275 2024-11-18T07:52:59,894 INFO [Thread-629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-629. 2024-11-18T07:52:59,894 DEBUG [Thread-629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:59,894 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:59,895 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:59,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:59,925 INFO [Thread-623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-623. 2024-11-18T07:52:59,925 DEBUG [Thread-623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:52:59,925 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:59,925 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:59,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T07:52:59,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-18T07:52:59,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-18T07:52:59,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-18T07:52:59,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-18T07:52:59,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-18T07:52:59,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T07:52:59,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2039 2024-11-18T07:52:59,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2210 2024-11-18T07:52:59,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T07:52:59,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 968 2024-11-18T07:52:59,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2901 rows 2024-11-18T07:52:59,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 962 2024-11-18T07:52:59,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2885 rows 2024-11-18T07:52:59,926 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T07:52:59,926 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T07:52:59,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:59,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:59,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:52:59,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:52:59,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T07:52:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T07:52:59,933 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916379933"}]},"ts":"1731916379933"} 2024-11-18T07:52:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T07:52:59,939 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f3b11bb24dc0fc107b3c002b8506511#C#compaction#182 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:52:59,939 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/8d2d8d39f42c4a63b6c495f134644fc5 is 50, key is test_row_0/C:col10/1731916376306/Put/seqid=0 2024-11-18T07:52:59,941 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T07:52:59,941 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T07:52:59,942 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T07:52:59,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f3b11bb24dc0fc107b3c002b8506511, UNASSIGN}] 2024-11-18T07:52:59,946 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f3b11bb24dc0fc107b3c002b8506511, UNASSIGN 2024-11-18T07:52:59,949 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=3f3b11bb24dc0fc107b3c002b8506511, regionState=CLOSING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:52:59,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f3b11bb24dc0fc107b3c002b8506511, UNASSIGN because future has completed 2024-11-18T07:52:59,955 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T07:52:59,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:52:59,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742047_1223 (size=13119) 2024-11-18T07:53:00,023 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/8d2d8d39f42c4a63b6c495f134644fc5 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/8d2d8d39f42c4a63b6c495f134644fc5 2024-11-18T07:53:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T07:53:00,067 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/C of 3f3b11bb24dc0fc107b3c002b8506511 into 8d2d8d39f42c4a63b6c495f134644fc5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:00,067 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:53:00,067 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/C, priority=13, startTime=1731916379517; duration=0sec 2024-11-18T07:53:00,067 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:00,067 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:C 2024-11-18T07:53:00,087 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/d4dc402086c0438f8512857c6dc673a3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/d4dc402086c0438f8512857c6dc673a3 2024-11-18T07:53:00,111 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:00,111 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T07:53:00,111 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing 3f3b11bb24dc0fc107b3c002b8506511, disabling compactions & flushes 2024-11-18T07:53:00,111 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:53:00,119 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f3b11bb24dc0fc107b3c002b8506511/B of 3f3b11bb24dc0fc107b3c002b8506511 into d4dc402086c0438f8512857c6dc673a3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:00,119 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f3b11bb24dc0fc107b3c002b8506511: 2024-11-18T07:53:00,119 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., storeName=3f3b11bb24dc0fc107b3c002b8506511/B, priority=13, startTime=1731916379516; duration=0sec 2024-11-18T07:53:00,119 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:53:00,119 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:53:00,119 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:00,119 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. after waiting 0 ms 2024-11-18T07:53:00,119 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f3b11bb24dc0fc107b3c002b8506511:B 2024-11-18T07:53:00,119 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:53:00,119 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing 3f3b11bb24dc0fc107b3c002b8506511 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-18T07:53:00,119 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=A 2024-11-18T07:53:00,119 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:00,120 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=B 2024-11-18T07:53:00,120 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:00,120 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f3b11bb24dc0fc107b3c002b8506511, store=C 2024-11-18T07:53:00,120 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:00,173 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184a85ded52bb64b9cb373aab368d4242b_3f3b11bb24dc0fc107b3c002b8506511 is 50, key is test_row_0/A:col10/1731916379893/Put/seqid=0 2024-11-18T07:53:00,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742048_1224 (size=12454) 2024-11-18T07:53:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T07:53:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T07:53:00,599 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:00,604 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184a85ded52bb64b9cb373aab368d4242b_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184a85ded52bb64b9cb373aab368d4242b_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:00,606 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/6203797a11e24883a91ed369162ad231, store: [table=TestAcidGuarantees family=A region=3f3b11bb24dc0fc107b3c002b8506511] 2024-11-18T07:53:00,606 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/6203797a11e24883a91ed369162ad231 is 175, key is test_row_0/A:col10/1731916379893/Put/seqid=0 2024-11-18T07:53:00,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742049_1225 (size=31255) 2024-11-18T07:53:01,025 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=453, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/6203797a11e24883a91ed369162ad231 2024-11-18T07:53:01,034 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/d14d741694874d6fbc8f69d8ea96c47a is 50, key is test_row_0/B:col10/1731916379893/Put/seqid=0 2024-11-18T07:53:01,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742050_1226 (size=12301) 2024-11-18T07:53:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T07:53:01,235 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T07:53:01,447 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/d14d741694874d6fbc8f69d8ea96c47a 2024-11-18T07:53:01,463 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/b86513deade841c1b7e0255263a95abd is 50, key is test_row_0/C:col10/1731916379893/Put/seqid=0 2024-11-18T07:53:01,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742051_1227 (size=12301) 2024-11-18T07:53:01,482 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/b86513deade841c1b7e0255263a95abd 2024-11-18T07:53:01,490 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/A/6203797a11e24883a91ed369162ad231 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/6203797a11e24883a91ed369162ad231 2024-11-18T07:53:01,496 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/6203797a11e24883a91ed369162ad231, entries=150, sequenceid=453, filesize=30.5 K 2024-11-18T07:53:01,499 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/B/d14d741694874d6fbc8f69d8ea96c47a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/d14d741694874d6fbc8f69d8ea96c47a 2024-11-18T07:53:01,506 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/d14d741694874d6fbc8f69d8ea96c47a, entries=150, sequenceid=453, filesize=12.0 K 2024-11-18T07:53:01,507 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/.tmp/C/b86513deade841c1b7e0255263a95abd as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/b86513deade841c1b7e0255263a95abd 2024-11-18T07:53:01,514 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/b86513deade841c1b7e0255263a95abd, entries=150, sequenceid=453, filesize=12.0 K 2024-11-18T07:53:01,519 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 3f3b11bb24dc0fc107b3c002b8506511 in 1399ms, sequenceid=453, compaction requested=false 2024-11-18T07:53:01,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/da4c05bf64724fc9b326aa78d05650d6, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/5f9bcb5614bc4cf691353c3ad8a98bc2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/90c1225a8f0e406bb4c87f035424be07, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/65acce3b2f6243f394b4d40beceb8333, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/0c1d8274bb354362a119ed503cc9f4e9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/d1168931e4dd43468c715f706f8708de, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/acea7b8087b44ba280bf0d7ceb86301b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/83e0361b33e14ad09a504e4bed61cc10, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/97fe9bb4056c4891bb382c585de16861, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a822452e855d4d9daf33b360f2e782fe, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/1fa21e47b7924dd98d07b4919970c7d3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a58c7455016748789881af024878b4cf, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/5c556dd4e6f84187aa21900eb9648404, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/ade21564eef3405488bc72a7c6f1941b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/3be051213876435cb019a72697de9880, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a5e3c7b07c2b40fdb29ee0f24cc98e16, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/dd5a4ab7df044a759e7b22e9eb57bd6f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e48f828d083046e2b36d6ea1dfe6b43e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/191da0f1a0774b2994f0d2e34929c3da, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e5b77fef5c9341e2a4d4333372f56a69, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/7616cd87bf3a4d8fb4f3f5e7fc027c96, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/d953fa8ac74e4b33aa163695b148bc57, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/f683050905d4454990b17fd903be46e0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/75b98e12be784401a8f3fd55bdbbe706] to archive 2024-11-18T07:53:01,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:53:01,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/da4c05bf64724fc9b326aa78d05650d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/da4c05bf64724fc9b326aa78d05650d6 2024-11-18T07:53:01,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/5f9bcb5614bc4cf691353c3ad8a98bc2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/5f9bcb5614bc4cf691353c3ad8a98bc2 2024-11-18T07:53:01,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/90c1225a8f0e406bb4c87f035424be07 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/90c1225a8f0e406bb4c87f035424be07 2024-11-18T07:53:01,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/65acce3b2f6243f394b4d40beceb8333 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/65acce3b2f6243f394b4d40beceb8333 2024-11-18T07:53:01,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/0c1d8274bb354362a119ed503cc9f4e9 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/0c1d8274bb354362a119ed503cc9f4e9 2024-11-18T07:53:01,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/d1168931e4dd43468c715f706f8708de to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/d1168931e4dd43468c715f706f8708de 2024-11-18T07:53:01,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/acea7b8087b44ba280bf0d7ceb86301b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/acea7b8087b44ba280bf0d7ceb86301b 2024-11-18T07:53:01,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/83e0361b33e14ad09a504e4bed61cc10 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/83e0361b33e14ad09a504e4bed61cc10 2024-11-18T07:53:01,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/97fe9bb4056c4891bb382c585de16861 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/97fe9bb4056c4891bb382c585de16861 2024-11-18T07:53:01,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a822452e855d4d9daf33b360f2e782fe to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a822452e855d4d9daf33b360f2e782fe 2024-11-18T07:53:01,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/1fa21e47b7924dd98d07b4919970c7d3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/1fa21e47b7924dd98d07b4919970c7d3 2024-11-18T07:53:01,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a58c7455016748789881af024878b4cf to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a58c7455016748789881af024878b4cf 2024-11-18T07:53:01,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/5c556dd4e6f84187aa21900eb9648404 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/5c556dd4e6f84187aa21900eb9648404 2024-11-18T07:53:01,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/ade21564eef3405488bc72a7c6f1941b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/ade21564eef3405488bc72a7c6f1941b 2024-11-18T07:53:01,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/3be051213876435cb019a72697de9880 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/3be051213876435cb019a72697de9880 2024-11-18T07:53:01,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a5e3c7b07c2b40fdb29ee0f24cc98e16 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/a5e3c7b07c2b40fdb29ee0f24cc98e16 2024-11-18T07:53:01,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/dd5a4ab7df044a759e7b22e9eb57bd6f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/dd5a4ab7df044a759e7b22e9eb57bd6f 2024-11-18T07:53:01,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e48f828d083046e2b36d6ea1dfe6b43e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e48f828d083046e2b36d6ea1dfe6b43e 2024-11-18T07:53:01,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/191da0f1a0774b2994f0d2e34929c3da to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/191da0f1a0774b2994f0d2e34929c3da 2024-11-18T07:53:01,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e5b77fef5c9341e2a4d4333372f56a69 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e5b77fef5c9341e2a4d4333372f56a69 2024-11-18T07:53:01,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/7616cd87bf3a4d8fb4f3f5e7fc027c96 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/7616cd87bf3a4d8fb4f3f5e7fc027c96 2024-11-18T07:53:01,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/d953fa8ac74e4b33aa163695b148bc57 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/d953fa8ac74e4b33aa163695b148bc57 2024-11-18T07:53:01,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/f683050905d4454990b17fd903be46e0 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/f683050905d4454990b17fd903be46e0 2024-11-18T07:53:01,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/75b98e12be784401a8f3fd55bdbbe706 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/75b98e12be784401a8f3fd55bdbbe706 2024-11-18T07:53:01,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/c3033bc0fae64daba9ea06132cf3aa97, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/31989abfd61f4d0e8aaafca3e9b6d511, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/2b60ba4b7fde4fda93ed64e202ebfc06, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/a0ec93e408744d3db84cf0b516472ab3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/426f99ec4e5848688477da001f7a92b0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/b7326b919cdd40b899038a4c452a927f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/c273cad045eb45918c1b852161e30192, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/07e5b87d8ba64dbb829baac8f7feedbb, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/53c6f1a50a0e442eb5630fdd60173017, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/d4f6646eb9a44a51af248137556742fe, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/8d38a5786a63441aa0b914d86c802ece, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/e004941d36ab4dd19f6eae3cd7f152be, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/aeea1f07f12b4b1d9e8bec6d8b996496, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/221e48ae87674239a558545950ce58c5, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/8ece6014f84842c7ae29299c4fa667d6, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/542551d8b8ac47e38eec7d846bd52b3e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/4d1b1d127ba9496f927ff518b66ae0e3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/f902b414beb14ca4b8ff85a2d517dd2d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/7047155a6d0445adbe45fd590bd4d638, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/1102ddf2cffd45afa05c9144d414c5b1, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/6304b2be87d840d497df340435a6ce69, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/e574baf85d904acda6694dc4d0b62e9a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/a412436f49254b40bce53b0a99639f97, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/0a8a0cc135f241deaa8641808da8dc51] to archive 2024-11-18T07:53:01,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:53:01,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/c3033bc0fae64daba9ea06132cf3aa97 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/c3033bc0fae64daba9ea06132cf3aa97 2024-11-18T07:53:01,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/31989abfd61f4d0e8aaafca3e9b6d511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/31989abfd61f4d0e8aaafca3e9b6d511 2024-11-18T07:53:01,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/2b60ba4b7fde4fda93ed64e202ebfc06 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/2b60ba4b7fde4fda93ed64e202ebfc06 2024-11-18T07:53:01,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/a0ec93e408744d3db84cf0b516472ab3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/a0ec93e408744d3db84cf0b516472ab3 2024-11-18T07:53:01,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/426f99ec4e5848688477da001f7a92b0 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/426f99ec4e5848688477da001f7a92b0 2024-11-18T07:53:01,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/b7326b919cdd40b899038a4c452a927f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/b7326b919cdd40b899038a4c452a927f 2024-11-18T07:53:01,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/c273cad045eb45918c1b852161e30192 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/c273cad045eb45918c1b852161e30192 2024-11-18T07:53:01,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/07e5b87d8ba64dbb829baac8f7feedbb to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/07e5b87d8ba64dbb829baac8f7feedbb 2024-11-18T07:53:01,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/53c6f1a50a0e442eb5630fdd60173017 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/53c6f1a50a0e442eb5630fdd60173017 2024-11-18T07:53:01,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/d4f6646eb9a44a51af248137556742fe to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/d4f6646eb9a44a51af248137556742fe 2024-11-18T07:53:01,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/8d38a5786a63441aa0b914d86c802ece to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/8d38a5786a63441aa0b914d86c802ece 2024-11-18T07:53:01,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/e004941d36ab4dd19f6eae3cd7f152be to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/e004941d36ab4dd19f6eae3cd7f152be 2024-11-18T07:53:01,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/aeea1f07f12b4b1d9e8bec6d8b996496 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/aeea1f07f12b4b1d9e8bec6d8b996496 2024-11-18T07:53:01,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/221e48ae87674239a558545950ce58c5 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/221e48ae87674239a558545950ce58c5 2024-11-18T07:53:01,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/8ece6014f84842c7ae29299c4fa667d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/8ece6014f84842c7ae29299c4fa667d6 2024-11-18T07:53:01,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/542551d8b8ac47e38eec7d846bd52b3e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/542551d8b8ac47e38eec7d846bd52b3e 2024-11-18T07:53:01,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/4d1b1d127ba9496f927ff518b66ae0e3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/4d1b1d127ba9496f927ff518b66ae0e3 2024-11-18T07:53:01,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/f902b414beb14ca4b8ff85a2d517dd2d to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/f902b414beb14ca4b8ff85a2d517dd2d 2024-11-18T07:53:01,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/7047155a6d0445adbe45fd590bd4d638 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/7047155a6d0445adbe45fd590bd4d638 2024-11-18T07:53:01,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/1102ddf2cffd45afa05c9144d414c5b1 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/1102ddf2cffd45afa05c9144d414c5b1 2024-11-18T07:53:01,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/6304b2be87d840d497df340435a6ce69 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/6304b2be87d840d497df340435a6ce69 2024-11-18T07:53:01,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/e574baf85d904acda6694dc4d0b62e9a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/e574baf85d904acda6694dc4d0b62e9a 2024-11-18T07:53:01,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/a412436f49254b40bce53b0a99639f97 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/a412436f49254b40bce53b0a99639f97 2024-11-18T07:53:01,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/0a8a0cc135f241deaa8641808da8dc51 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/0a8a0cc135f241deaa8641808da8dc51 2024-11-18T07:53:01,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/bef40946d3e64ed681f2d1aa9aa4cdd0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/027e21a481264f4ead3d2046764406ed, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/d3b2afb6ae3a45489de5707c29ee4646, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/1f30b7fca59a4acdba430a296e636ce4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/ed6c3752a8c549fdaf9b640a3beaf503, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0e0da02e1cb749fbaeaed57686cfdefe, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/1ccfc07f1ea9474fa45d4e00341999c6, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/f8dace3717cd4113b28bad9057453c10, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/8d103e618f53481c895c071945d50703, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/d49dd181f6b74529a9136e903c1e317e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/9af01dde2e2d4e70b41388bcd8901109, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0d0b61a273ba4f0ab3ad1bda925d7bb3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/106e1b84527c42059ad1fed92ae208ff, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/040494a6877a4b84be6b4e39da4383ba, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/78c25ca19c4645d29e90a40bf4be14d2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/755b8765ac6047acb682e2feab2d7f70, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/78e18ff1e2d64164a92a7d77626b37f4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/b23e9a883b194680b7bc90b8c140aca7, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/c810b12e0a274997a2e445d792e1e4da, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/366ab0a10db54791a253befc8636ea50, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0321a56ef452499bbf49e06b2e27f6d2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/392ed8b0b31946a9a984d2486d08aeed, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/fc2357d6a9b74a2d806f20633686e9df, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/23ce947f014945249b65c6df15e36711] to archive 2024-11-18T07:53:01,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:53:01,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/bef40946d3e64ed681f2d1aa9aa4cdd0 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/bef40946d3e64ed681f2d1aa9aa4cdd0 2024-11-18T07:53:01,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/027e21a481264f4ead3d2046764406ed to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/027e21a481264f4ead3d2046764406ed 2024-11-18T07:53:01,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/d3b2afb6ae3a45489de5707c29ee4646 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/d3b2afb6ae3a45489de5707c29ee4646 2024-11-18T07:53:01,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/1f30b7fca59a4acdba430a296e636ce4 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/1f30b7fca59a4acdba430a296e636ce4 2024-11-18T07:53:01,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/ed6c3752a8c549fdaf9b640a3beaf503 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/ed6c3752a8c549fdaf9b640a3beaf503 2024-11-18T07:53:01,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0e0da02e1cb749fbaeaed57686cfdefe to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0e0da02e1cb749fbaeaed57686cfdefe 2024-11-18T07:53:01,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/1ccfc07f1ea9474fa45d4e00341999c6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/1ccfc07f1ea9474fa45d4e00341999c6 2024-11-18T07:53:01,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/f8dace3717cd4113b28bad9057453c10 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/f8dace3717cd4113b28bad9057453c10 2024-11-18T07:53:01,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/8d103e618f53481c895c071945d50703 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/8d103e618f53481c895c071945d50703 2024-11-18T07:53:01,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/d49dd181f6b74529a9136e903c1e317e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/d49dd181f6b74529a9136e903c1e317e 2024-11-18T07:53:01,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/9af01dde2e2d4e70b41388bcd8901109 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/9af01dde2e2d4e70b41388bcd8901109 2024-11-18T07:53:01,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0d0b61a273ba4f0ab3ad1bda925d7bb3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0d0b61a273ba4f0ab3ad1bda925d7bb3 2024-11-18T07:53:01,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/106e1b84527c42059ad1fed92ae208ff to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/106e1b84527c42059ad1fed92ae208ff 2024-11-18T07:53:01,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/040494a6877a4b84be6b4e39da4383ba to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/040494a6877a4b84be6b4e39da4383ba 2024-11-18T07:53:01,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/78c25ca19c4645d29e90a40bf4be14d2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/78c25ca19c4645d29e90a40bf4be14d2 2024-11-18T07:53:01,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/755b8765ac6047acb682e2feab2d7f70 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/755b8765ac6047acb682e2feab2d7f70 2024-11-18T07:53:01,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/78e18ff1e2d64164a92a7d77626b37f4 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/78e18ff1e2d64164a92a7d77626b37f4 2024-11-18T07:53:01,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/b23e9a883b194680b7bc90b8c140aca7 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/b23e9a883b194680b7bc90b8c140aca7 2024-11-18T07:53:01,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/c810b12e0a274997a2e445d792e1e4da to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/c810b12e0a274997a2e445d792e1e4da 2024-11-18T07:53:01,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/366ab0a10db54791a253befc8636ea50 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/366ab0a10db54791a253befc8636ea50 2024-11-18T07:53:01,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0321a56ef452499bbf49e06b2e27f6d2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/0321a56ef452499bbf49e06b2e27f6d2 2024-11-18T07:53:01,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/392ed8b0b31946a9a984d2486d08aeed to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/392ed8b0b31946a9a984d2486d08aeed 2024-11-18T07:53:01,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/fc2357d6a9b74a2d806f20633686e9df to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/fc2357d6a9b74a2d806f20633686e9df 2024-11-18T07:53:01,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/23ce947f014945249b65c6df15e36711 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/23ce947f014945249b65c6df15e36711 2024-11-18T07:53:01,760 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/recovered.edits/456.seqid, newMaxSeqId=456, maxSeqId=4 2024-11-18T07:53:01,761 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. 2024-11-18T07:53:01,762 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for 3f3b11bb24dc0fc107b3c002b8506511: Waiting for close lock at 1731916380111Running coprocessor pre-close hooks at 1731916380111Disabling compacts and flushes for region at 1731916380111Disabling writes for close at 1731916380119 (+8 ms)Obtaining lock to block concurrent updates at 1731916380119Preparing flush snapshotting stores in 3f3b11bb24dc0fc107b3c002b8506511 at 1731916380119Finished memstore snapshotting TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731916380120 (+1 ms)Flushing stores of TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511. at 1731916380123 (+3 ms)Flushing 3f3b11bb24dc0fc107b3c002b8506511/A: creating writer at 1731916380123Flushing mob file 3f3b11bb24dc0fc107b3c002b8506511/A: appending metadata at 1731916380173 (+50 ms)Flushing mob file 3f3b11bb24dc0fc107b3c002b8506511/A: closing flushed file at 1731916380173Flushing 3f3b11bb24dc0fc107b3c002b8506511/A: appending metadata at 1731916380606 (+433 ms)Flushing 3f3b11bb24dc0fc107b3c002b8506511/A: closing flushed file at 1731916380606Flushing 3f3b11bb24dc0fc107b3c002b8506511/B: creating writer at 1731916381030 (+424 ms)Flushing 3f3b11bb24dc0fc107b3c002b8506511/B: appending metadata at 1731916381034 (+4 ms)Flushing 3f3b11bb24dc0fc107b3c002b8506511/B: closing flushed file at 1731916381034Flushing 3f3b11bb24dc0fc107b3c002b8506511/C: creating writer at 1731916381453 (+419 ms)Flushing 3f3b11bb24dc0fc107b3c002b8506511/C: appending metadata at 1731916381463 (+10 ms)Flushing 3f3b11bb24dc0fc107b3c002b8506511/C: closing flushed file at 1731916381463Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6185a92f: reopening flushed file at 1731916381489 (+26 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5df49a4d: reopening flushed file at 1731916381496 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@353426e0: reopening flushed file at 1731916381506 (+10 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 3f3b11bb24dc0fc107b3c002b8506511 in 1399ms, sequenceid=453, compaction requested=false at 1731916381519 (+13 ms)Writing region close event to WAL at 1731916381746 (+227 ms)Running coprocessor post-close hooks at 1731916381761 (+15 ms)Closed at 1731916381761 2024-11-18T07:53:01,768 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed 3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:01,769 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=3f3b11bb24dc0fc107b3c002b8506511, regionState=CLOSED 2024-11-18T07:53:01,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:53:01,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-18T07:53:01,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure 3f3b11bb24dc0fc107b3c002b8506511, server=87fb3d252950,37987,1731916323128 in 1.8170 sec 2024-11-18T07:53:01,776 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-18T07:53:01,776 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f3b11bb24dc0fc107b3c002b8506511, UNASSIGN in 1.8290 sec 2024-11-18T07:53:01,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-18T07:53:01,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8350 sec 2024-11-18T07:53:01,780 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916381779"}]},"ts":"1731916381779"} 2024-11-18T07:53:01,781 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T07:53:01,781 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T07:53:01,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8540 sec 2024-11-18T07:53:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T07:53:02,089 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:02,089 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T07:53:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:02,092 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T07:53:02,094 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:02,100 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,103 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/recovered.edits] 2024-11-18T07:53:02,106 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/6203797a11e24883a91ed369162ad231 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/6203797a11e24883a91ed369162ad231 2024-11-18T07:53:02,107 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e99b27296ac447bd92f19b252a05a347 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/A/e99b27296ac447bd92f19b252a05a347 2024-11-18T07:53:02,109 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/d14d741694874d6fbc8f69d8ea96c47a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/d14d741694874d6fbc8f69d8ea96c47a 2024-11-18T07:53:02,111 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/d4dc402086c0438f8512857c6dc673a3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/B/d4dc402086c0438f8512857c6dc673a3 2024-11-18T07:53:02,114 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/8d2d8d39f42c4a63b6c495f134644fc5 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/8d2d8d39f42c4a63b6c495f134644fc5 2024-11-18T07:53:02,115 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/b86513deade841c1b7e0255263a95abd to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/C/b86513deade841c1b7e0255263a95abd 2024-11-18T07:53:02,118 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/recovered.edits/456.seqid to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511/recovered.edits/456.seqid 2024-11-18T07:53:02,119 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,119 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T07:53:02,120 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T07:53:02,121 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T07:53:02,125 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118049e8d85d01f4321b5328f12f2d1b4c8_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118049e8d85d01f4321b5328f12f2d1b4c8_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,128 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111824ad543dad564d5588fb8e81e34dff72_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111824ad543dad564d5588fb8e81e34dff72_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,130 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111843069f46bf314bb7b577a5a5a4875353_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111843069f46bf314bb7b577a5a5a4875353_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,132 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111843aabe3429aa411396daaca279033a3d_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111843aabe3429aa411396daaca279033a3d_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,133 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184a85ded52bb64b9cb373aab368d4242b_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184a85ded52bb64b9cb373aab368d4242b_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,135 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184b38513660074890ae589890991dd4a3_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184b38513660074890ae589890991dd4a3_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,136 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185b06fdcb7c4345f28cca7989d85b2642_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185b06fdcb7c4345f28cca7989d85b2642_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,137 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187c6984d2b29849b1a629403057a7a4c6_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187c6984d2b29849b1a629403057a7a4c6_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,139 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111898c0aa6d615a457d9552c598c36d3ad6_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111898c0aa6d615a457d9552c598c36d3ad6_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,140 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b1786aa2cccd4b92a391e0951dc481f8_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b1786aa2cccd4b92a391e0951dc481f8_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,141 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b2ff4149e3d547b2b6a89f9d80f92512_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b2ff4149e3d547b2b6a89f9d80f92512_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,142 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b762c385198149ed9adfc1c9ce8c6873_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b762c385198149ed9adfc1c9ce8c6873_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,144 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b8228bbe55374535b41f09dd2e8e0fb0_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b8228bbe55374535b41f09dd2e8e0fb0_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,145 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118be12e8efac74453e8e62124f16f7fb6f_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118be12e8efac74453e8e62124f16f7fb6f_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,146 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c7ea9577b6a747089dc76e8518f2f2b8_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c7ea9577b6a747089dc76e8518f2f2b8_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,147 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d123181b064c42069e08a9d7d9588c92_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d123181b064c42069e08a9d7d9588c92_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,148 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d5ff3d29c4b44aaf9319ca32e68c3975_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d5ff3d29c4b44aaf9319ca32e68c3975_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,150 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e338728c4ca3460e8f6babdf02a2d32e_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e338728c4ca3460e8f6babdf02a2d32e_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,151 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fb154e68d6f748c7be73b09e37646c16_3f3b11bb24dc0fc107b3c002b8506511 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fb154e68d6f748c7be73b09e37646c16_3f3b11bb24dc0fc107b3c002b8506511 2024-11-18T07:53:02,151 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T07:53:02,154 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:02,156 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T07:53:02,160 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T07:53:02,162 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:02,162 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T07:53:02,162 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731916382162"}]},"ts":"9223372036854775807"} 2024-11-18T07:53:02,164 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T07:53:02,165 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3f3b11bb24dc0fc107b3c002b8506511, NAME => 'TestAcidGuarantees,,1731916353292.3f3b11bb24dc0fc107b3c002b8506511.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T07:53:02,165 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T07:53:02,165 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731916382165"}]},"ts":"9223372036854775807"} 2024-11-18T07:53:02,167 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T07:53:02,168 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:02,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 79 msec 2024-11-18T07:53:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T07:53:02,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T07:53:02,199 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:02,209 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=253 (was 249) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1820827738_22 at /127.0.0.1:56402 [Waiting for operation #165] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1820827738_22 at /127.0.0.1:41718 [Waiting for operation #360] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/cluster_477ae96c-b03e-42a8-3641-251bd626575f/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-349956365_22 at /127.0.0.1:54082 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-349956365_22 at /127.0.0.1:56238 [Waiting for operation #229] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-349956365_22 at /127.0.0.1:39124 [Waiting for operation #106] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/cluster_477ae96c-b03e-42a8-3641-251bd626575f/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=463 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=733 (was 666) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2591 (was 415) - AvailableMemoryMB LEAK? - 2024-11-18T07:53:02,218 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=253, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=733, ProcessCount=11, AvailableMemoryMB=2588 2024-11-18T07:53:02,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T07:53:02,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T07:53:02,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:02,222 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T07:53:02,222 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:02,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-11-18T07:53:02,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-18T07:53:02,224 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T07:53:02,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742052_1228 (size=1061) 2024-11-18T07:53:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-18T07:53:02,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-18T07:53:02,640 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3 2024-11-18T07:53:02,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742053_1229 (size=53) 2024-11-18T07:53:02,725 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T07:53:02,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-18T07:53:03,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:53:03,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1e0c954a4c9c2df78d79f9a8a402493e, disabling compactions & flushes 2024-11-18T07:53:03,072 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:03,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:03,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. after waiting 0 ms 2024-11-18T07:53:03,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:03,072 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:03,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1e0c954a4c9c2df78d79f9a8a402493e: Waiting for close lock at 1731916383072Disabling compacts and flushes for region at 1731916383072Disabling writes for close at 1731916383072Writing region close event to WAL at 1731916383072Closed at 1731916383072 2024-11-18T07:53:03,074 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T07:53:03,074 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731916383074"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731916383074"}]},"ts":"1731916383074"} 2024-11-18T07:53:03,076 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T07:53:03,077 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T07:53:03,077 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916383077"}]},"ts":"1731916383077"} 2024-11-18T07:53:03,079 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T07:53:03,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e0c954a4c9c2df78d79f9a8a402493e, ASSIGN}] 2024-11-18T07:53:03,080 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e0c954a4c9c2df78d79f9a8a402493e, ASSIGN 2024-11-18T07:53:03,081 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e0c954a4c9c2df78d79f9a8a402493e, ASSIGN; state=OFFLINE, location=87fb3d252950,37987,1731916323128; forceNewPlan=false, retain=false 2024-11-18T07:53:03,232 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=1e0c954a4c9c2df78d79f9a8a402493e, regionState=OPENING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:53:03,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e0c954a4c9c2df78d79f9a8a402493e, ASSIGN because future has completed 2024-11-18T07:53:03,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:53:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-18T07:53:03,395 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:03,396 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} 2024-11-18T07:53:03,396 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:03,396 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:53:03,396 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:03,396 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:03,398 INFO [StoreOpener-1e0c954a4c9c2df78d79f9a8a402493e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:03,400 INFO [StoreOpener-1e0c954a4c9c2df78d79f9a8a402493e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:53:03,400 INFO [StoreOpener-1e0c954a4c9c2df78d79f9a8a402493e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e0c954a4c9c2df78d79f9a8a402493e columnFamilyName A 2024-11-18T07:53:03,400 DEBUG [StoreOpener-1e0c954a4c9c2df78d79f9a8a402493e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:03,405 INFO [StoreOpener-1e0c954a4c9c2df78d79f9a8a402493e-1 {}] regionserver.HStore(327): Store=1e0c954a4c9c2df78d79f9a8a402493e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:53:03,405 INFO [StoreOpener-1e0c954a4c9c2df78d79f9a8a402493e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:03,407 INFO [StoreOpener-1e0c954a4c9c2df78d79f9a8a402493e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:53:03,408 INFO [StoreOpener-1e0c954a4c9c2df78d79f9a8a402493e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e0c954a4c9c2df78d79f9a8a402493e columnFamilyName B 2024-11-18T07:53:03,408 DEBUG [StoreOpener-1e0c954a4c9c2df78d79f9a8a402493e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:03,414 INFO [StoreOpener-1e0c954a4c9c2df78d79f9a8a402493e-1 {}] regionserver.HStore(327): Store=1e0c954a4c9c2df78d79f9a8a402493e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:53:03,414 INFO [StoreOpener-1e0c954a4c9c2df78d79f9a8a402493e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:03,425 INFO [StoreOpener-1e0c954a4c9c2df78d79f9a8a402493e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:53:03,425 INFO [StoreOpener-1e0c954a4c9c2df78d79f9a8a402493e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e0c954a4c9c2df78d79f9a8a402493e columnFamilyName C 2024-11-18T07:53:03,425 DEBUG [StoreOpener-1e0c954a4c9c2df78d79f9a8a402493e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:03,426 INFO [StoreOpener-1e0c954a4c9c2df78d79f9a8a402493e-1 {}] regionserver.HStore(327): Store=1e0c954a4c9c2df78d79f9a8a402493e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:53:03,426 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:03,426 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:03,428 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:03,429 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:03,430 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:03,430 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:03,430 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T07:53:03,432 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:03,444 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T07:53:03,445 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened 1e0c954a4c9c2df78d79f9a8a402493e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61711367, jitterRate=-0.08042897284030914}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T07:53:03,445 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:03,445 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for 1e0c954a4c9c2df78d79f9a8a402493e: Running coprocessor pre-open hook at 1731916383397Writing region info on filesystem at 1731916383397Initializing all the Stores at 1731916383398 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916383398Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916383398Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916383398Cleaning up temporary data from old regions at 1731916383430 (+32 ms)Running coprocessor post-open hooks at 1731916383445 (+15 ms)Region opened successfully at 1731916383445 2024-11-18T07:53:03,448 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., pid=57, masterSystemTime=1731916383388 2024-11-18T07:53:03,450 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:03,450 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:03,451 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=1e0c954a4c9c2df78d79f9a8a402493e, regionState=OPEN, openSeqNum=2, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:53:03,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:53:03,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-18T07:53:03,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure 1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 in 220 msec 2024-11-18T07:53:03,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-18T07:53:03,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e0c954a4c9c2df78d79f9a8a402493e, ASSIGN in 383 msec 2024-11-18T07:53:03,468 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T07:53:03,468 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916383468"}]},"ts":"1731916383468"} 2024-11-18T07:53:03,470 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T07:53:03,472 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T07:53:03,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2520 sec 2024-11-18T07:53:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-18T07:53:04,381 INFO [Async-Client-Retry-Timer-pool-0 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:04,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33ebe564, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:04,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:04,388 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cda7895, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:04,392 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:04,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d590e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,393 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:04,398 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:04,400 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T07:53:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 87fb3d252950,42633,1731916322278 2024-11-18T07:53:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2bfa211e 2024-11-18T07:53:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T07:53:04,404 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T07:53:04,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279bf308, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,410 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:04,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:04,411 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@433f537a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:04,414 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:04,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@140b17b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ba0760, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:04,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:04,418 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@246f4289, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:04,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:04,427 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:04,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc7aea0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@102b3c4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:04,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:04,436 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@313a21a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:04,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:04,440 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e410428, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@574b1ec0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:04,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:04,446 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:04,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@523d8f82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:04,449 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:04,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a08084, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ea361c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,481 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:04,481 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:04,487 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2055d37a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:04,490 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:04,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f61452e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c85ada2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:04,495 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:04,496 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a62f61b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:04,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:04,503 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f5b2d7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3660e239, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,515 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:04,515 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:04,517 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:04,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:04,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:04,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15bf2458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:04,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:04,520 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fb28b47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:04,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67353b2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:04,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:04,539 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10cbf99e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:04,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:04,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:04,541 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b40e651, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df95348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,559 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:04,559 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:04,560 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e12abb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:04,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:04,564 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:04,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@346609c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@edffef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:04,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:04,580 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@498a2c33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:04,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:04,586 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:04,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@778579dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:04,603 DEBUG [Thread-1075 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:04,607 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:04,611 DEBUG [Thread-1077 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:04,614 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:04,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-18T07:53:04,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T07:53:04,627 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:04,628 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:04,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:04,635 DEBUG [Thread-1083 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:04,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T07:53:04,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:04,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:04,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:04,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:04,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:04,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:04,636 DEBUG [Thread-1083 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:04,636 DEBUG [Thread-1083 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:04,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:04,640 DEBUG [Thread-1083 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:04,652 DEBUG [Thread-1086 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:04,656 DEBUG [Thread-1086 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:04,656 DEBUG [Thread-1086 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:04,659 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:04,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916444691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:04,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916444693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:04,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:04,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916444692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:04,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:04,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916384676, max timestamp=1731916384689Num uniques 859; 2024-11-18T07:53:04,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916384676, max timestamp=1731916384689Num uniques 859; 2024-11-18T07:53:04,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731916384676, max timestamp=1731916384689Num uniques 859; 2024-11-18T07:53:04,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:53:04,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:53:04,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:53:04,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:04,708 DEBUG [Thread-1090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:04,711 DEBUG [Thread-1081 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:04,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:04,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:04,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T07:53:04,729 DEBUG [Thread-1094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:04,730 DEBUG [Thread-1094 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:04,731 DEBUG [Thread-1094 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:04,732 DEBUG [Thread-1079 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:04,733 DEBUG [Thread-1079 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:04,734 DEBUG [Thread-1079 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:04,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:04,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:04,738 DEBUG [Thread-1092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:04,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:04,741 DEBUG [Thread-1092 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:04,741 DEBUG [Thread-1092 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:04,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:04,747 DEBUG [Thread-1088 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:04,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:04,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:04,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:04,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/c5d8a4d9478a4fc1977a8407330a650d is 50, key is test_row_0/A:col10/1731916384624/Put/seqid=0 2024-11-18T07:53:04,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-18T07:53:04,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:04,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:04,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:04,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:04,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:04,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916444823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:04,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:04,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916444827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:04,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916444829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:04,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:04,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916444838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:04,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:04,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916444840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:04,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742054_1230 (size=12001) 2024-11-18T07:53:04,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T07:53:04,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-18T07:53:04,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:04,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:04,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:04,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:04,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:04,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:04,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916444950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:04,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916444950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:04,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916444951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:04,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:04,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916444969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:04,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916444969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:05,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-18T07:53:05,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:05,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:05,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:05,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916445181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:05,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:05,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916445183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916445183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:05,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916445199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:05,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916445199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T07:53:05,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-18T07:53:05,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:05,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:05,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:05,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/c5d8a4d9478a4fc1977a8407330a650d 2024-11-18T07:53:05,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/4ac1bfb6e0274fc9904f9db01cf9517b is 50, key is test_row_0/B:col10/1731916384624/Put/seqid=0 2024-11-18T07:53:05,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-18T07:53:05,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:05,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:05,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:05,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742055_1231 (size=12001) 2024-11-18T07:53:05,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/4ac1bfb6e0274fc9904f9db01cf9517b 2024-11-18T07:53:05,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916445508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:05,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916445508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:05,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916445515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:05,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916445518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:05,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:05,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916445518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:05,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:05,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:05,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/3109ca67c3f14d109e2fff7a724c903b is 50, key is test_row_0/C:col10/1731916384624/Put/seqid=0 2024-11-18T07:53:05,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-18T07:53:05,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:05,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:05,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:05,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742056_1232 (size=12001) 2024-11-18T07:53:05,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-18T07:53:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:05,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T07:53:05,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-18T07:53:05,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:05,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:05,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:05,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:06,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/3109ca67c3f14d109e2fff7a724c903b 2024-11-18T07:53:06,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916446028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:06,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916446028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:06,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916446039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:06,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916446039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:06,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916446043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:06,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/c5d8a4d9478a4fc1977a8407330a650d as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/c5d8a4d9478a4fc1977a8407330a650d 2024-11-18T07:53:06,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-18T07:53:06,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:06,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:06,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:06,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:06,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:06,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/c5d8a4d9478a4fc1977a8407330a650d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-18T07:53:06,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/4ac1bfb6e0274fc9904f9db01cf9517b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/4ac1bfb6e0274fc9904f9db01cf9517b 2024-11-18T07:53:06,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/4ac1bfb6e0274fc9904f9db01cf9517b, entries=150, sequenceid=12, filesize=11.7 K 2024-11-18T07:53:06,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/3109ca67c3f14d109e2fff7a724c903b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/3109ca67c3f14d109e2fff7a724c903b 2024-11-18T07:53:06,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/3109ca67c3f14d109e2fff7a724c903b, entries=150, sequenceid=12, filesize=11.7 K 2024-11-18T07:53:06,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1e0c954a4c9c2df78d79f9a8a402493e in 1599ms, sequenceid=12, compaction requested=false 2024-11-18T07:53:06,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:06,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-18T07:53:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:06,259 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-18T07:53:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:06,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/f52abba34ce943f89bcd1d6f14ee4435 is 50, key is test_row_0/A:col10/1731916384827/Put/seqid=0 2024-11-18T07:53:06,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742057_1233 (size=12001) 2024-11-18T07:53:06,383 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/f52abba34ce943f89bcd1d6f14ee4435 2024-11-18T07:53:06,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/7666878fdfff4f8fac362828872c4ecd is 50, key is test_row_0/B:col10/1731916384827/Put/seqid=0 2024-11-18T07:53:06,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742058_1234 (size=12001) 2024-11-18T07:53:06,480 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/7666878fdfff4f8fac362828872c4ecd 2024-11-18T07:53:06,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/7969631957e047a4865b76c155ba362e is 50, key is test_row_0/C:col10/1731916384827/Put/seqid=0 2024-11-18T07:53:06,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742059_1235 (size=12001) 2024-11-18T07:53:06,560 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/7969631957e047a4865b76c155ba362e 2024-11-18T07:53:06,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/f52abba34ce943f89bcd1d6f14ee4435 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/f52abba34ce943f89bcd1d6f14ee4435 2024-11-18T07:53:06,592 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/f52abba34ce943f89bcd1d6f14ee4435, entries=150, sequenceid=47, filesize=11.7 K 2024-11-18T07:53:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/7666878fdfff4f8fac362828872c4ecd as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/7666878fdfff4f8fac362828872c4ecd 2024-11-18T07:53:06,621 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/7666878fdfff4f8fac362828872c4ecd, entries=150, sequenceid=47, filesize=11.7 K 2024-11-18T07:53:06,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/7969631957e047a4865b76c155ba362e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/7969631957e047a4865b76c155ba362e 2024-11-18T07:53:06,668 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/7969631957e047a4865b76c155ba362e, entries=150, sequenceid=47, filesize=11.7 K 2024-11-18T07:53:06,670 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=0 B/0 for 1e0c954a4c9c2df78d79f9a8a402493e in 411ms, sequenceid=47, compaction requested=false 2024-11-18T07:53:06,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:06,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:06,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-18T07:53:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-18T07:53:06,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-18T07:53:06,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0590 sec 2024-11-18T07:53:06,699 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.0800 sec 2024-11-18T07:53:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T07:53:06,778 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:06,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-18T07:53:06,781 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:06,783 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:06,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T07:53:06,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T07:53:06,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T07:53:06,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:06,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:06,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:06,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-18T07:53:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-18T07:53:06,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-18T07:53:06,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 167 msec 2024-11-18T07:53:06,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 180 msec 2024-11-18T07:53:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:07,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T07:53:07,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:07,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:07,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:07,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:07,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:07,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T07:53:07,099 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:07,100 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:07,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/e8d12613070d420eb520f74773486a44 is 50, key is test_row_0/A:col10/1731916387060/Put/seqid=0 2024-11-18T07:53:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-18T07:53:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T07:53:07,115 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:07,116 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:07,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:07,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742060_1236 (size=12001) 2024-11-18T07:53:07,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916447179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:07,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916447187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:07,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:07,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916447191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T07:53:07,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916447207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:07,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916447211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:07,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:07,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T07:53:07,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:07,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:07,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:07,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:07,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:07,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916447318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916447318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:07,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916447318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:07,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916447338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:07,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916447340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:07,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:07,355 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T07:53:07,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T07:53:07,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:07,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:07,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:07,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:07,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T07:53:07,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/e8d12613070d420eb520f74773486a44 2024-11-18T07:53:07,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:07,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916447544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:07,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916447544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:07,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916447549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:07,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/af61d337779f42d5b67a2264272854e2 is 50, key is test_row_0/B:col10/1731916387060/Put/seqid=0 2024-11-18T07:53:07,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916447555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:07,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916447559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:07,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:07,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:07,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:07,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T07:53:07,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:07,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:07,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:07,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:07,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:07,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742061_1237 (size=12001) 2024-11-18T07:53:07,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/af61d337779f42d5b67a2264272854e2 2024-11-18T07:53:07,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/321ef2ce32eb4d1395dbcfc9c3a06110 is 50, key is test_row_0/C:col10/1731916387060/Put/seqid=0 2024-11-18T07:53:07,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T07:53:07,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:07,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:07,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:07,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T07:53:07,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:07,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:07,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742062_1238 (size=12001) 2024-11-18T07:53:07,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/321ef2ce32eb4d1395dbcfc9c3a06110 2024-11-18T07:53:07,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/e8d12613070d420eb520f74773486a44 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/e8d12613070d420eb520f74773486a44 2024-11-18T07:53:07,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/e8d12613070d420eb520f74773486a44, entries=150, sequenceid=60, filesize=11.7 K 2024-11-18T07:53:07,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/af61d337779f42d5b67a2264272854e2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/af61d337779f42d5b67a2264272854e2 2024-11-18T07:53:07,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/af61d337779f42d5b67a2264272854e2, entries=150, sequenceid=60, filesize=11.7 K 2024-11-18T07:53:07,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/321ef2ce32eb4d1395dbcfc9c3a06110 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/321ef2ce32eb4d1395dbcfc9c3a06110 2024-11-18T07:53:07,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/321ef2ce32eb4d1395dbcfc9c3a06110, entries=150, sequenceid=60, filesize=11.7 K 2024-11-18T07:53:07,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1e0c954a4c9c2df78d79f9a8a402493e in 719ms, sequenceid=60, compaction requested=true 2024-11-18T07:53:07,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:07,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:53:07,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:07,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:53:07,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:53:07,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:53:07,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T07:53:07,798 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:07,798 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:07,804 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:07,804 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/A is initiating minor compaction (all files) 2024-11-18T07:53:07,804 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/A in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:07,804 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/c5d8a4d9478a4fc1977a8407330a650d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/f52abba34ce943f89bcd1d6f14ee4435, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/e8d12613070d420eb520f74773486a44] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=35.2 K 2024-11-18T07:53:07,808 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:07,808 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/C is initiating minor compaction (all files) 2024-11-18T07:53:07,808 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/C in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:07,809 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/3109ca67c3f14d109e2fff7a724c903b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/7969631957e047a4865b76c155ba362e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/321ef2ce32eb4d1395dbcfc9c3a06110] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=35.2 K 2024-11-18T07:53:07,809 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5d8a4d9478a4fc1977a8407330a650d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731916384624 2024-11-18T07:53:07,809 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3109ca67c3f14d109e2fff7a724c903b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731916384624 2024-11-18T07:53:07,809 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f52abba34ce943f89bcd1d6f14ee4435, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731916384827 2024-11-18T07:53:07,809 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7969631957e047a4865b76c155ba362e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731916384827 2024-11-18T07:53:07,810 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8d12613070d420eb520f74773486a44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916387060 2024-11-18T07:53:07,810 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 321ef2ce32eb4d1395dbcfc9c3a06110, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916387060 2024-11-18T07:53:07,857 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#A#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:07,863 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/c335ea01db874b07aef640a153661c8a is 50, key is test_row_0/A:col10/1731916387060/Put/seqid=0 2024-11-18T07:53:07,877 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#C#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:07,877 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/2dc0dda9575745dbabd06bbe7f7e2aa2 is 50, key is test_row_0/C:col10/1731916387060/Put/seqid=0 2024-11-18T07:53:07,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916387166, max timestamp=1731916387884Num uniques 859; 2024-11-18T07:53:07,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:53:07,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=102.87 KB heapSize=301.95 KB 2024-11-18T07:53:07,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:07,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:07,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:07,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:07,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:07,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:07,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:07,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T07:53:07,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:07,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:07,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:07,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:07,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:07,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:07,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742063_1239 (size=12104) 2024-11-18T07:53:07,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/bc205bbfe5f849438d15b77c3a98d083 is 50, key is test_row_0/A:col10/1731916387884/Put/seqid=0 2024-11-18T07:53:07,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742064_1240 (size=12104) 2024-11-18T07:53:08,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916447973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,007 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/2dc0dda9575745dbabd06bbe7f7e2aa2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/2dc0dda9575745dbabd06bbe7f7e2aa2 2024-11-18T07:53:08,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916447975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916447995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742065_1241 (size=12001) 2024-11-18T07:53:08,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/bc205bbfe5f849438d15b77c3a98d083 2024-11-18T07:53:08,023 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/C of 1e0c954a4c9c2df78d79f9a8a402493e into 2dc0dda9575745dbabd06bbe7f7e2aa2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:08,023 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:08,023 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/C, priority=13, startTime=1731916387798; duration=0sec 2024-11-18T07:53:08,023 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:08,023 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:C 2024-11-18T07:53:08,023 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:08,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916448007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916448007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,035 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:08,035 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/B is initiating minor compaction (all files) 2024-11-18T07:53:08,035 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/B in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:08,035 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/4ac1bfb6e0274fc9904f9db01cf9517b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/7666878fdfff4f8fac362828872c4ecd, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/af61d337779f42d5b67a2264272854e2] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=35.2 K 2024-11-18T07:53:08,040 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ac1bfb6e0274fc9904f9db01cf9517b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731916384624 2024-11-18T07:53:08,043 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7666878fdfff4f8fac362828872c4ecd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731916384827 2024-11-18T07:53:08,048 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting af61d337779f42d5b67a2264272854e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916387060 2024-11-18T07:53:08,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T07:53:08,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:08,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:08,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:08,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/eb19e409f1f447a1a68d3c5fafbc6d92 is 50, key is test_row_0/B:col10/1731916387884/Put/seqid=0 2024-11-18T07:53:08,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916448119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916448119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916448119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916448138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,156 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#B#compaction#199 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:08,157 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/5728298c272e4131ab2aa18960a8373b is 50, key is test_row_0/B:col10/1731916387060/Put/seqid=0 2024-11-18T07:53:08,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742066_1242 (size=12001) 2024-11-18T07:53:08,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916448179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742067_1243 (size=12104) 2024-11-18T07:53:08,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T07:53:08,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:08,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:08,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:08,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T07:53:08,251 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/5728298c272e4131ab2aa18960a8373b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/5728298c272e4131ab2aa18960a8373b 2024-11-18T07:53:08,302 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/B of 1e0c954a4c9c2df78d79f9a8a402493e into 5728298c272e4131ab2aa18960a8373b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:08,302 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:08,302 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/B, priority=13, startTime=1731916387798; duration=0sec 2024-11-18T07:53:08,303 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:08,303 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:B 2024-11-18T07:53:08,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916448348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916448348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916448359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916448368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T07:53:08,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:08,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:08,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:08,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916448398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,435 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/c335ea01db874b07aef640a153661c8a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/c335ea01db874b07aef640a153661c8a 2024-11-18T07:53:08,511 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/A of 1e0c954a4c9c2df78d79f9a8a402493e into c335ea01db874b07aef640a153661c8a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:08,511 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:08,511 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/A, priority=13, startTime=1731916387798; duration=0sec 2024-11-18T07:53:08,511 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:08,511 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:A 2024-11-18T07:53:08,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T07:53:08,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:08,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:08,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:08,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/eb19e409f1f447a1a68d3c5fafbc6d92 2024-11-18T07:53:08,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/86dcddb137214e918e086b042d895679 is 50, key is test_row_0/C:col10/1731916387884/Put/seqid=0 2024-11-18T07:53:08,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916448668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916448669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742068_1244 (size=12001) 2024-11-18T07:53:08,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916448678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916448689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916448718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:08,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T07:53:08,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:08,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:08,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:08,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T07:53:08,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:08,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:08,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:08,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:09,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T07:53:09,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:09,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:09,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:09,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:09,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:09,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/86dcddb137214e918e086b042d895679 2024-11-18T07:53:09,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/bc205bbfe5f849438d15b77c3a98d083 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/bc205bbfe5f849438d15b77c3a98d083 2024-11-18T07:53:09,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/bc205bbfe5f849438d15b77c3a98d083, entries=150, sequenceid=83, filesize=11.7 K 2024-11-18T07:53:09,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T07:53:09,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:09,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:09,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:09,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:09,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:09,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/eb19e409f1f447a1a68d3c5fafbc6d92 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/eb19e409f1f447a1a68d3c5fafbc6d92 2024-11-18T07:53:09,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916449199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:09,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916449199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:09,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916449219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:09,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916449219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:09,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:09,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:09,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:09,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:09,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:09,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:09,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/eb19e409f1f447a1a68d3c5fafbc6d92, entries=150, sequenceid=83, filesize=11.7 K 2024-11-18T07:53:09,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/86dcddb137214e918e086b042d895679 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/86dcddb137214e918e086b042d895679 2024-11-18T07:53:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T07:53:09,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916449238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:09,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:09,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:09,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:09,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/86dcddb137214e918e086b042d895679, entries=150, sequenceid=83, filesize=11.7 K 2024-11-18T07:53:09,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.87 KB/105340, heapSize ~301.91 KB/309152, currentSize=80.51 KB/82440 for 1e0c954a4c9c2df78d79f9a8a402493e in 1394ms, sequenceid=83, compaction requested=false 2024-11-18T07:53:09,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:09,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T07:53:09,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:09,378 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T07:53:09,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:09,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/06fe1e4e2271476293b8a2f2de80b835 is 50, key is test_row_0/A:col10/1731916387968/Put/seqid=0 2024-11-18T07:53:09,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742069_1245 (size=12001) 2024-11-18T07:53:09,871 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/06fe1e4e2271476293b8a2f2de80b835 2024-11-18T07:53:09,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/ec5a735dce374bcf976cd1275e0f05f3 is 50, key is test_row_0/B:col10/1731916387968/Put/seqid=0 2024-11-18T07:53:09,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742070_1246 (size=12001) 2024-11-18T07:53:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:10,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:10,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916450288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916450298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916450299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916450300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916450303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,357 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/ec5a735dce374bcf976cd1275e0f05f3 2024-11-18T07:53:10,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/5d851db5453c46a4886208a157a13e03 is 50, key is test_row_0/C:col10/1731916387968/Put/seqid=0 2024-11-18T07:53:10,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916450411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916450412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916450412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916450412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916450419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742071_1247 (size=12001) 2024-11-18T07:53:10,443 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/5d851db5453c46a4886208a157a13e03 2024-11-18T07:53:10,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/06fe1e4e2271476293b8a2f2de80b835 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/06fe1e4e2271476293b8a2f2de80b835 2024-11-18T07:53:10,474 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/06fe1e4e2271476293b8a2f2de80b835, entries=150, sequenceid=101, filesize=11.7 K 2024-11-18T07:53:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/ec5a735dce374bcf976cd1275e0f05f3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ec5a735dce374bcf976cd1275e0f05f3 2024-11-18T07:53:10,487 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ec5a735dce374bcf976cd1275e0f05f3, entries=150, sequenceid=101, filesize=11.7 K 2024-11-18T07:53:10,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/5d851db5453c46a4886208a157a13e03 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5d851db5453c46a4886208a157a13e03 2024-11-18T07:53:10,543 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5d851db5453c46a4886208a157a13e03, entries=150, sequenceid=101, filesize=11.7 K 2024-11-18T07:53:10,544 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1e0c954a4c9c2df78d79f9a8a402493e in 1166ms, sequenceid=101, compaction requested=true 2024-11-18T07:53:10,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:10,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:10,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-18T07:53:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-18T07:53:10,550 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-18T07:53:10,550 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4290 sec 2024-11-18T07:53:10,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 3.4510 sec 2024-11-18T07:53:10,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T07:53:10,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:10,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:10,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:10,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:10,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:10,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:10,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/929eafc549fe4436b6f35a1f49dad675 is 50, key is test_row_0/A:col10/1731916390622/Put/seqid=0 2024-11-18T07:53:10,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916450683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916450687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916450689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916450694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916450695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742072_1248 (size=16681) 2024-11-18T07:53:10,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916450798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916450811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916450818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916450828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:10,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916450829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:11,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916451018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:11,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:11,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916451038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:11,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:11,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916451039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:11,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916451039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:11,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916451039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:11,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/929eafc549fe4436b6f35a1f49dad675 2024-11-18T07:53:11,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/6fdb8b68257a46e6b9a38f73ae9f892f is 50, key is test_row_0/B:col10/1731916390622/Put/seqid=0 2024-11-18T07:53:11,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742073_1249 (size=12001) 2024-11-18T07:53:11,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/6fdb8b68257a46e6b9a38f73ae9f892f 2024-11-18T07:53:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T07:53:11,284 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:11,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-18T07:53:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T07:53:11,293 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:11,295 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:11,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:11,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/883fd27cc96d48148e9f22b9cd266c84 is 50, key is test_row_0/C:col10/1731916390622/Put/seqid=0 2024-11-18T07:53:11,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742074_1250 (size=12001) 2024-11-18T07:53:11,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916451339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:11,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:11,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916451367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:11,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916451368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:11,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916451371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:11,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916451372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:11,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T07:53:11,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-18T07:53:11,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:11,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:11,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:11,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:11,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:11,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:11,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-18T07:53:11,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:11,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:11,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:11,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T07:53:11,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:11,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/883fd27cc96d48148e9f22b9cd266c84 2024-11-18T07:53:11,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-18T07:53:11,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:11,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:11,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:11,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:11,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:11,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/929eafc549fe4436b6f35a1f49dad675 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/929eafc549fe4436b6f35a1f49dad675 2024-11-18T07:53:11,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916451858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:11,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/929eafc549fe4436b6f35a1f49dad675, entries=250, sequenceid=120, filesize=16.3 K 2024-11-18T07:53:11,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/6fdb8b68257a46e6b9a38f73ae9f892f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/6fdb8b68257a46e6b9a38f73ae9f892f 2024-11-18T07:53:11,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916451893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:11,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916451898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:11,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916451899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/6fdb8b68257a46e6b9a38f73ae9f892f, entries=150, sequenceid=120, filesize=11.7 K 2024-11-18T07:53:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:11,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916451893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:11,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:11,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/883fd27cc96d48148e9f22b9cd266c84 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/883fd27cc96d48148e9f22b9cd266c84 2024-11-18T07:53:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T07:53:11,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/883fd27cc96d48148e9f22b9cd266c84, entries=150, sequenceid=120, filesize=11.7 K 2024-11-18T07:53:11,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1e0c954a4c9c2df78d79f9a8a402493e in 1296ms, sequenceid=120, compaction requested=true 2024-11-18T07:53:11,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-18T07:53:11,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:11,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:53:11,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:11,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:11,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:53:11,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:53:11,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:53:11,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T07:53:11,943 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T07:53:11,943 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:53:11,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:11,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:11,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:11,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:11,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:11,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:11,946 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:53:11,948 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:53:11,948 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/C is initiating minor compaction (all files) 2024-11-18T07:53:11,949 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/C in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:11,949 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/2dc0dda9575745dbabd06bbe7f7e2aa2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/86dcddb137214e918e086b042d895679, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5d851db5453c46a4886208a157a13e03, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/883fd27cc96d48148e9f22b9cd266c84] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=47.0 K 2024-11-18T07:53:11,949 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dc0dda9575745dbabd06bbe7f7e2aa2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916387060 2024-11-18T07:53:11,950 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86dcddb137214e918e086b042d895679, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731916387205 2024-11-18T07:53:11,950 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d851db5453c46a4886208a157a13e03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731916387968 2024-11-18T07:53:11,950 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 883fd27cc96d48148e9f22b9cd266c84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731916390298 2024-11-18T07:53:11,955 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:53:11,955 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/A is initiating minor compaction (all files) 2024-11-18T07:53:11,955 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/A in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:11,956 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/c335ea01db874b07aef640a153661c8a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/bc205bbfe5f849438d15b77c3a98d083, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/06fe1e4e2271476293b8a2f2de80b835, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/929eafc549fe4436b6f35a1f49dad675] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=51.5 K 2024-11-18T07:53:11,958 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting c335ea01db874b07aef640a153661c8a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916387060 2024-11-18T07:53:11,959 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting bc205bbfe5f849438d15b77c3a98d083, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731916387205 2024-11-18T07:53:11,960 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 06fe1e4e2271476293b8a2f2de80b835, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731916387968 2024-11-18T07:53:11,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/6c01dd4abed84e63a914195f6c1f9aab is 50, key is test_row_0/A:col10/1731916390688/Put/seqid=0 2024-11-18T07:53:11,965 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 929eafc549fe4436b6f35a1f49dad675, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731916390284 2024-11-18T07:53:11,989 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#C#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:11,990 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/2a41b1f7fa20497c87409535393a868d is 50, key is test_row_0/C:col10/1731916390622/Put/seqid=0 2024-11-18T07:53:12,014 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#A#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:12,015 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/36e2de44a84b4c539d8435846198fb14 is 50, key is test_row_0/A:col10/1731916390622/Put/seqid=0 2024-11-18T07:53:12,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742075_1251 (size=12151) 2024-11-18T07:53:12,051 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/6c01dd4abed84e63a914195f6c1f9aab 2024-11-18T07:53:12,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742076_1252 (size=12241) 2024-11-18T07:53:12,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742077_1253 (size=12241) 2024-11-18T07:53:12,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/cbef7c0379f54bff93777153763127f2 is 50, key is test_row_0/B:col10/1731916390688/Put/seqid=0 2024-11-18T07:53:12,098 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/2a41b1f7fa20497c87409535393a868d as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/2a41b1f7fa20497c87409535393a868d 2024-11-18T07:53:12,115 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/C of 1e0c954a4c9c2df78d79f9a8a402493e into 2a41b1f7fa20497c87409535393a868d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:12,115 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:12,115 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/C, priority=12, startTime=1731916391943; duration=0sec 2024-11-18T07:53:12,115 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:12,115 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:C 2024-11-18T07:53:12,116 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:53:12,120 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:53:12,120 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/B is initiating minor compaction (all files) 2024-11-18T07:53:12,120 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/B in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:12,120 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/5728298c272e4131ab2aa18960a8373b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/eb19e409f1f447a1a68d3c5fafbc6d92, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ec5a735dce374bcf976cd1275e0f05f3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/6fdb8b68257a46e6b9a38f73ae9f892f] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=47.0 K 2024-11-18T07:53:12,122 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5728298c272e4131ab2aa18960a8373b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916387060 2024-11-18T07:53:12,123 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/36e2de44a84b4c539d8435846198fb14 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/36e2de44a84b4c539d8435846198fb14 2024-11-18T07:53:12,124 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb19e409f1f447a1a68d3c5fafbc6d92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731916387205 2024-11-18T07:53:12,124 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec5a735dce374bcf976cd1275e0f05f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731916387968 2024-11-18T07:53:12,129 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fdb8b68257a46e6b9a38f73ae9f892f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731916390298 2024-11-18T07:53:12,140 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/A of 1e0c954a4c9c2df78d79f9a8a402493e into 36e2de44a84b4c539d8435846198fb14(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:12,140 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:12,140 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/A, priority=12, startTime=1731916391943; duration=0sec 2024-11-18T07:53:12,140 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:12,140 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:A 2024-11-18T07:53:12,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742078_1254 (size=12151) 2024-11-18T07:53:12,195 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#B#compaction#211 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:12,195 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/e697fe8c853c448a84730488c26b9379 is 50, key is test_row_0/B:col10/1731916390622/Put/seqid=0 2024-11-18T07:53:12,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742079_1255 (size=12241) 2024-11-18T07:53:12,274 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/e697fe8c853c448a84730488c26b9379 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/e697fe8c853c448a84730488c26b9379 2024-11-18T07:53:12,299 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/B of 1e0c954a4c9c2df78d79f9a8a402493e into e697fe8c853c448a84730488c26b9379(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:12,299 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:12,299 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/B, priority=12, startTime=1731916391943; duration=0sec 2024-11-18T07:53:12,299 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:12,299 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:B 2024-11-18T07:53:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T07:53:12,555 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/cbef7c0379f54bff93777153763127f2 2024-11-18T07:53:12,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/0ddf5320cb0547d9a3271ec55a5171c8 is 50, key is test_row_0/C:col10/1731916390688/Put/seqid=0 2024-11-18T07:53:12,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742080_1256 (size=12151) 2024-11-18T07:53:12,727 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T07:53:12,727 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-18T07:53:12,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:12,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:12,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916452970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:12,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916452981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:12,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916452983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916452988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916452999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,082 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/0ddf5320cb0547d9a3271ec55a5171c8 2024-11-18T07:53:13,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/6c01dd4abed84e63a914195f6c1f9aab as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/6c01dd4abed84e63a914195f6c1f9aab 2024-11-18T07:53:13,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916453098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,107 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/6c01dd4abed84e63a914195f6c1f9aab, entries=150, sequenceid=133, filesize=11.9 K 2024-11-18T07:53:13,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/cbef7c0379f54bff93777153763127f2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/cbef7c0379f54bff93777153763127f2 2024-11-18T07:53:13,124 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/cbef7c0379f54bff93777153763127f2, entries=150, sequenceid=133, filesize=11.9 K 2024-11-18T07:53:13,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/0ddf5320cb0547d9a3271ec55a5171c8 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0ddf5320cb0547d9a3271ec55a5171c8 2024-11-18T07:53:13,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916453109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916453109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916453121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916453121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,134 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0ddf5320cb0547d9a3271ec55a5171c8, entries=150, sequenceid=133, filesize=11.9 K 2024-11-18T07:53:13,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,136 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 1e0c954a4c9c2df78d79f9a8a402493e in 1192ms, sequenceid=133, compaction requested=false 2024-11-18T07:53:13,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:13,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:13,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-18T07:53:13,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-18T07:53:13,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-18T07:53:13,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8430 sec 2024-11-18T07:53:13,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.8530 sec 2024-11-18T07:53:13,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916392968, max timestamp=1731916393320Num uniques 859; 2024-11-18T07:53:13,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:53:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:13,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-18T07:53:13,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:13,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:13,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:13,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:13,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:13,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:13,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/a3bee9ca8f4b44f4938f9d16f8f1fb2b is 50, key is test_row_0/A:col10/1731916392968/Put/seqid=0 2024-11-18T07:53:13,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742081_1257 (size=14541) 2024-11-18T07:53:13,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/a3bee9ca8f4b44f4938f9d16f8f1fb2b 2024-11-18T07:53:13,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916453372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916453390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/0ca85685ef0047f989ddfe3c01fd6f12 is 50, key is test_row_0/B:col10/1731916392968/Put/seqid=0 2024-11-18T07:53:13,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916453411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916453399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916453427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T07:53:13,451 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:13,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-18T07:53:13,454 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:13,456 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:13,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T07:53:13,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742082_1258 (size=12151) 2024-11-18T07:53:13,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916453528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916453529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916453538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916453538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916453538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T07:53:13,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T07:53:13,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:13,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:13,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:13,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:13,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:13,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916453738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916453738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916453749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916453749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916453749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:13,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T07:53:13,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T07:53:13,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:13,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:13,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:13,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/0ca85685ef0047f989ddfe3c01fd6f12 2024-11-18T07:53:13,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/5b6104d2c5f64f5cae175018db039a6f is 50, key is test_row_0/C:col10/1731916392968/Put/seqid=0 2024-11-18T07:53:13,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742083_1259 (size=12151) 2024-11-18T07:53:13,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T07:53:13,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:13,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:13,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:13,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:13,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:13,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:14,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916454051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916454063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916454071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916454071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916454084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T07:53:14,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T07:53:14,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:14,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:14,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:14,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:14,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:14,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T07:53:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:14,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:14,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:14,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:14,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:14,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/5b6104d2c5f64f5cae175018db039a6f 2024-11-18T07:53:14,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/a3bee9ca8f4b44f4938f9d16f8f1fb2b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/a3bee9ca8f4b44f4938f9d16f8f1fb2b 2024-11-18T07:53:14,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/a3bee9ca8f4b44f4938f9d16f8f1fb2b, entries=200, sequenceid=157, filesize=14.2 K 2024-11-18T07:53:14,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/0ca85685ef0047f989ddfe3c01fd6f12 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/0ca85685ef0047f989ddfe3c01fd6f12 2024-11-18T07:53:14,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/0ca85685ef0047f989ddfe3c01fd6f12, entries=150, sequenceid=157, filesize=11.9 K 2024-11-18T07:53:14,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/5b6104d2c5f64f5cae175018db039a6f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5b6104d2c5f64f5cae175018db039a6f 2024-11-18T07:53:14,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5b6104d2c5f64f5cae175018db039a6f, entries=150, sequenceid=157, filesize=11.9 K 2024-11-18T07:53:14,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 1e0c954a4c9c2df78d79f9a8a402493e in 1071ms, sequenceid=157, compaction requested=true 2024-11-18T07:53:14,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:14,397 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:14,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:53:14,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:14,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:53:14,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:14,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:53:14,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:53:14,400 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:14,400 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/A is initiating minor compaction (all files) 2024-11-18T07:53:14,400 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/A in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:14,400 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/36e2de44a84b4c539d8435846198fb14, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/6c01dd4abed84e63a914195f6c1f9aab, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/a3bee9ca8f4b44f4938f9d16f8f1fb2b] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=38.0 K 2024-11-18T07:53:14,401 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:14,402 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:14,402 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/B is initiating minor compaction (all files) 2024-11-18T07:53:14,402 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/B in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:14,403 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/e697fe8c853c448a84730488c26b9379, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/cbef7c0379f54bff93777153763127f2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/0ca85685ef0047f989ddfe3c01fd6f12] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=35.7 K 2024-11-18T07:53:14,403 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e697fe8c853c448a84730488c26b9379, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731916390298 2024-11-18T07:53:14,403 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 36e2de44a84b4c539d8435846198fb14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731916390298 2024-11-18T07:53:14,403 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbef7c0379f54bff93777153763127f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731916390669 2024-11-18T07:53:14,404 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ca85685ef0047f989ddfe3c01fd6f12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731916392968 2024-11-18T07:53:14,405 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c01dd4abed84e63a914195f6c1f9aab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731916390669 2024-11-18T07:53:14,407 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting a3bee9ca8f4b44f4938f9d16f8f1fb2b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731916392968 2024-11-18T07:53:14,422 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#B#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:14,422 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/0d2c8db7186a469c98131adb3c365f63 is 50, key is test_row_0/B:col10/1731916392968/Put/seqid=0 2024-11-18T07:53:14,428 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#A#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:14,429 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/8f3b62e6ebdf41e3861cac6b44d526f7 is 50, key is test_row_0/A:col10/1731916392968/Put/seqid=0 2024-11-18T07:53:14,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T07:53:14,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:14,463 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T07:53:14,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:14,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:14,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:14,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:14,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:14,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:14,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742084_1260 (size=12493) 2024-11-18T07:53:14,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/917fd4f86cba4ac0834af5d3d5ee4431 is 50, key is test_row_0/A:col10/1731916393336/Put/seqid=0 2024-11-18T07:53:14,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742085_1261 (size=12493) 2024-11-18T07:53:14,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742086_1262 (size=12151) 2024-11-18T07:53:14,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T07:53:14,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916454613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916454614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916454621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916454622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916454624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916454728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916454728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916454738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916454738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916454738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,919 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/0d2c8db7186a469c98131adb3c365f63 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/0d2c8db7186a469c98131adb3c365f63 2024-11-18T07:53:14,939 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/B of 1e0c954a4c9c2df78d79f9a8a402493e into 0d2c8db7186a469c98131adb3c365f63(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:14,939 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:14,939 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/B, priority=13, startTime=1731916394398; duration=0sec 2024-11-18T07:53:14,939 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:14,939 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:B 2024-11-18T07:53:14,939 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:14,941 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/8f3b62e6ebdf41e3861cac6b44d526f7 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/8f3b62e6ebdf41e3861cac6b44d526f7 2024-11-18T07:53:14,942 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:14,942 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/C is initiating minor compaction (all files) 2024-11-18T07:53:14,942 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/C in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:14,942 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/2a41b1f7fa20497c87409535393a868d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0ddf5320cb0547d9a3271ec55a5171c8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5b6104d2c5f64f5cae175018db039a6f] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=35.7 K 2024-11-18T07:53:14,943 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a41b1f7fa20497c87409535393a868d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731916390298 2024-11-18T07:53:14,943 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ddf5320cb0547d9a3271ec55a5171c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731916390669 2024-11-18T07:53:14,944 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b6104d2c5f64f5cae175018db039a6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731916392968 2024-11-18T07:53:14,949 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/917fd4f86cba4ac0834af5d3d5ee4431 2024-11-18T07:53:14,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916454939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916454939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916454949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916454949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916454951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:14,969 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#C#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:14,970 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/5293ad29fa9d49be94c7eb64d23a8f8d is 50, key is test_row_0/C:col10/1731916392968/Put/seqid=0 2024-11-18T07:53:14,974 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/A of 1e0c954a4c9c2df78d79f9a8a402493e into 8f3b62e6ebdf41e3861cac6b44d526f7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:14,974 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:14,974 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/A, priority=13, startTime=1731916394397; duration=0sec 2024-11-18T07:53:14,975 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:14,975 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:A 2024-11-18T07:53:14,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/f2f8a53bd8704492abea70a5699d4cf0 is 50, key is test_row_0/B:col10/1731916393336/Put/seqid=0 2024-11-18T07:53:14,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742088_1264 (size=12151) 2024-11-18T07:53:15,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742087_1263 (size=12493) 2024-11-18T07:53:15,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916455268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:15,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:15,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916455269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:15,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916455269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:15,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916455270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:15,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916455270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:15,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:15,391 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/f2f8a53bd8704492abea70a5699d4cf0 2024-11-18T07:53:15,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/ac2c828add6a4df68f39c7b9ab24ae35 is 50, key is test_row_0/C:col10/1731916393336/Put/seqid=0 2024-11-18T07:53:15,430 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/5293ad29fa9d49be94c7eb64d23a8f8d as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5293ad29fa9d49be94c7eb64d23a8f8d 2024-11-18T07:53:15,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742089_1265 (size=12151) 2024-11-18T07:53:15,451 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/C of 1e0c954a4c9c2df78d79f9a8a402493e into 5293ad29fa9d49be94c7eb64d23a8f8d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:15,451 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:15,451 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/C, priority=13, startTime=1731916394398; duration=0sec 2024-11-18T07:53:15,451 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:15,451 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:C 2024-11-18T07:53:15,463 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/ac2c828add6a4df68f39c7b9ab24ae35 2024-11-18T07:53:15,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/917fd4f86cba4ac0834af5d3d5ee4431 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/917fd4f86cba4ac0834af5d3d5ee4431 2024-11-18T07:53:15,486 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/917fd4f86cba4ac0834af5d3d5ee4431, entries=150, sequenceid=173, filesize=11.9 K 2024-11-18T07:53:15,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/f2f8a53bd8704492abea70a5699d4cf0 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/f2f8a53bd8704492abea70a5699d4cf0 2024-11-18T07:53:15,495 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/f2f8a53bd8704492abea70a5699d4cf0, entries=150, sequenceid=173, filesize=11.9 K 2024-11-18T07:53:15,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/ac2c828add6a4df68f39c7b9ab24ae35 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/ac2c828add6a4df68f39c7b9ab24ae35 2024-11-18T07:53:15,503 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/ac2c828add6a4df68f39c7b9ab24ae35, entries=150, sequenceid=173, filesize=11.9 K 2024-11-18T07:53:15,507 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1e0c954a4c9c2df78d79f9a8a402493e in 1044ms, sequenceid=173, compaction requested=false 2024-11-18T07:53:15,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:15,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:15,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-18T07:53:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-18T07:53:15,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-18T07:53:15,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0550 sec 2024-11-18T07:53:15,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.0620 sec 2024-11-18T07:53:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T07:53:15,623 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:15,631 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-18T07:53:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T07:53:15,639 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:15,643 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:15,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:15,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T07:53:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:15,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T07:53:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:15,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T07:53:15,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:15,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:15,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:15,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:15,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:15,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/af74f5649e0c41d583905eaf81fd56f3 is 50, key is test_row_0/A:col10/1731916395793/Put/seqid=0 2024-11-18T07:53:15,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916455828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:15,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916455834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:15,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916455845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:15,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916455848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916455849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:15,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742090_1266 (size=12151) 2024-11-18T07:53:15,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/af74f5649e0c41d583905eaf81fd56f3 2024-11-18T07:53:15,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/07cb5a6839a5495e9ee62ef9cb5c2862 is 50, key is test_row_0/B:col10/1731916395793/Put/seqid=0 2024-11-18T07:53:15,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742091_1267 (size=12151) 2024-11-18T07:53:15,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/07cb5a6839a5495e9ee62ef9cb5c2862 2024-11-18T07:53:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T07:53:15,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T07:53:15,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:15,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:15,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:15,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:15,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:15,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:15,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916455958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:15,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916455958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916455958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916455968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:15,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916455969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:15,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/9f31e1a58d0e42deaf97884ab04619c6 is 50, key is test_row_0/C:col10/1731916395793/Put/seqid=0 2024-11-18T07:53:16,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742092_1268 (size=12151) 2024-11-18T07:53:16,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T07:53:16,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:16,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:16,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:16,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:16,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:16,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:16,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916456179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:16,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916456179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:16,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916456179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:16,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916456179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:16,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:16,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:16,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916456189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:16,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T07:53:16,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T07:53:16,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:16,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:16,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:16,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:16,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:16,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T07:53:16,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/9f31e1a58d0e42deaf97884ab04619c6 2024-11-18T07:53:16,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:16,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:16,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:16,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:16,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:16,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/af74f5649e0c41d583905eaf81fd56f3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/af74f5649e0c41d583905eaf81fd56f3 2024-11-18T07:53:16,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/af74f5649e0c41d583905eaf81fd56f3, entries=150, sequenceid=195, filesize=11.9 K 2024-11-18T07:53:16,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:16,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/07cb5a6839a5495e9ee62ef9cb5c2862 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/07cb5a6839a5495e9ee62ef9cb5c2862 2024-11-18T07:53:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916456488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:16,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916456488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:16,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916456499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:16,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916456499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:16,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916456500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:16,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/07cb5a6839a5495e9ee62ef9cb5c2862, entries=150, sequenceid=195, filesize=11.9 K 2024-11-18T07:53:16,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/9f31e1a58d0e42deaf97884ab04619c6 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/9f31e1a58d0e42deaf97884ab04619c6 2024-11-18T07:53:16,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/9f31e1a58d0e42deaf97884ab04619c6, entries=150, sequenceid=195, filesize=11.9 K 2024-11-18T07:53:16,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 1e0c954a4c9c2df78d79f9a8a402493e in 734ms, sequenceid=195, compaction requested=true 2024-11-18T07:53:16,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:16,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:53:16,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:16,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:53:16,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:53:16,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:53:16,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T07:53:16,530 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:16,533 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:16,536 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:16,536 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/A is initiating minor compaction (all files) 2024-11-18T07:53:16,536 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/A in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:16,537 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/8f3b62e6ebdf41e3861cac6b44d526f7, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/917fd4f86cba4ac0834af5d3d5ee4431, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/af74f5649e0c41d583905eaf81fd56f3] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=35.9 K 2024-11-18T07:53:16,537 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:16,537 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/C is initiating minor compaction (all files) 2024-11-18T07:53:16,537 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/C in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:16,537 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5293ad29fa9d49be94c7eb64d23a8f8d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/ac2c828add6a4df68f39c7b9ab24ae35, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/9f31e1a58d0e42deaf97884ab04619c6] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=35.9 K 2024-11-18T07:53:16,538 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f3b62e6ebdf41e3861cac6b44d526f7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731916392968 2024-11-18T07:53:16,538 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5293ad29fa9d49be94c7eb64d23a8f8d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731916392968 2024-11-18T07:53:16,538 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 917fd4f86cba4ac0834af5d3d5ee4431, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731916393336 2024-11-18T07:53:16,539 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting ac2c828add6a4df68f39c7b9ab24ae35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731916393336 2024-11-18T07:53:16,539 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting af74f5649e0c41d583905eaf81fd56f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731916395793 2024-11-18T07:53:16,540 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f31e1a58d0e42deaf97884ab04619c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731916395793 2024-11-18T07:53:16,565 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:16,566 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/4664e164f1714af6a85a085774e00018 is 50, key is test_row_0/A:col10/1731916395793/Put/seqid=0 2024-11-18T07:53:16,584 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#C#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:16,585 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/0000dcafcc5045b6b3d34e8b7b79c445 is 50, key is test_row_0/C:col10/1731916395793/Put/seqid=0 2024-11-18T07:53:16,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T07:53:16,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:16,601 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T07:53:16,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:16,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:16,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:16,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:16,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:16,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:16,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/15ee1f9e21e748b6ae558c985eb39cd2 is 50, key is test_row_0/A:col10/1731916395843/Put/seqid=0 2024-11-18T07:53:16,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742093_1269 (size=12595) 2024-11-18T07:53:16,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742094_1270 (size=12595) 2024-11-18T07:53:16,642 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/4664e164f1714af6a85a085774e00018 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/4664e164f1714af6a85a085774e00018 2024-11-18T07:53:16,650 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/0000dcafcc5045b6b3d34e8b7b79c445 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0000dcafcc5045b6b3d34e8b7b79c445 2024-11-18T07:53:16,651 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/A of 1e0c954a4c9c2df78d79f9a8a402493e into 4664e164f1714af6a85a085774e00018(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:16,651 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:16,651 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/A, priority=13, startTime=1731916396530; duration=0sec 2024-11-18T07:53:16,651 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:16,651 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:A 2024-11-18T07:53:16,651 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:16,655 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:16,655 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/B is initiating minor compaction (all files) 2024-11-18T07:53:16,655 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/B in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:16,655 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/0d2c8db7186a469c98131adb3c365f63, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/f2f8a53bd8704492abea70a5699d4cf0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/07cb5a6839a5495e9ee62ef9cb5c2862] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=35.9 K 2024-11-18T07:53:16,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742095_1271 (size=12151) 2024-11-18T07:53:16,659 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/C of 1e0c954a4c9c2df78d79f9a8a402493e into 0000dcafcc5045b6b3d34e8b7b79c445(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:16,660 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:16,660 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/C, priority=13, startTime=1731916396530; duration=0sec 2024-11-18T07:53:16,660 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:16,660 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:C 2024-11-18T07:53:16,660 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d2c8db7186a469c98131adb3c365f63, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731916392968 2024-11-18T07:53:16,661 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2f8a53bd8704492abea70a5699d4cf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731916393336 2024-11-18T07:53:16,661 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07cb5a6839a5495e9ee62ef9cb5c2862, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731916395793 2024-11-18T07:53:16,681 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#B#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:16,681 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/23044d9376c048178673f8091128b639 is 50, key is test_row_0/B:col10/1731916395793/Put/seqid=0 2024-11-18T07:53:16,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742096_1272 (size=12595) 2024-11-18T07:53:16,755 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/23044d9376c048178673f8091128b639 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/23044d9376c048178673f8091128b639 2024-11-18T07:53:16,761 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/B of 1e0c954a4c9c2df78d79f9a8a402493e into 23044d9376c048178673f8091128b639(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:16,761 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:16,761 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/B, priority=13, startTime=1731916396530; duration=0sec 2024-11-18T07:53:16,762 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:16,762 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:B 2024-11-18T07:53:16,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T07:53:17,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:17,062 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/15ee1f9e21e748b6ae558c985eb39cd2 2024-11-18T07:53:17,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916457055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916457064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916457064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916457067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916457068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/71a9fe6dcb1e4303b35f3cc77468122f is 50, key is test_row_0/B:col10/1731916395843/Put/seqid=0 2024-11-18T07:53:17,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742097_1273 (size=12151) 2024-11-18T07:53:17,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916457179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916457179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916457189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916457189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916457189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916457388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916457389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916457403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916457410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916457410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,585 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/71a9fe6dcb1e4303b35f3cc77468122f 2024-11-18T07:53:17,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/64d5706650b3426ab4aa9f720ceb0ee0 is 50, key is test_row_0/C:col10/1731916395843/Put/seqid=0 2024-11-18T07:53:17,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742098_1274 (size=12151) 2024-11-18T07:53:17,670 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/64d5706650b3426ab4aa9f720ceb0ee0 2024-11-18T07:53:17,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/15ee1f9e21e748b6ae558c985eb39cd2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/15ee1f9e21e748b6ae558c985eb39cd2 2024-11-18T07:53:17,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916457708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916457708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916457718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,736 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/15ee1f9e21e748b6ae558c985eb39cd2, entries=150, sequenceid=209, filesize=11.9 K 2024-11-18T07:53:17,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916457731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916457731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:17,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/71a9fe6dcb1e4303b35f3cc77468122f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/71a9fe6dcb1e4303b35f3cc77468122f 2024-11-18T07:53:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:17,784 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/71a9fe6dcb1e4303b35f3cc77468122f, entries=150, sequenceid=209, filesize=11.9 K 2024-11-18T07:53:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T07:53:17,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/64d5706650b3426ab4aa9f720ceb0ee0 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/64d5706650b3426ab4aa9f720ceb0ee0 2024-11-18T07:53:17,796 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/64d5706650b3426ab4aa9f720ceb0ee0, entries=150, sequenceid=209, filesize=11.9 K 2024-11-18T07:53:17,799 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1e0c954a4c9c2df78d79f9a8a402493e in 1198ms, sequenceid=209, compaction requested=false 2024-11-18T07:53:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-18T07:53:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-18T07:53:17,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-18T07:53:17,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1640 sec 2024-11-18T07:53:17,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.1800 sec 2024-11-18T07:53:18,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T07:53:18,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:18,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:18,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:18,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:18,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:18,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:18,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/953765d871134cbf902decd45b3cdfc2 is 50, key is test_row_0/A:col10/1731916397060/Put/seqid=0 2024-11-18T07:53:18,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916458286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916458306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916458307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916458310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916458312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742099_1275 (size=14541) 2024-11-18T07:53:18,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/953765d871134cbf902decd45b3cdfc2 2024-11-18T07:53:18,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/29928505f10747fdbe7dfd048f6fd8cd is 50, key is test_row_0/B:col10/1731916397060/Put/seqid=0 2024-11-18T07:53:18,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916458423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916458431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916458431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916458435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916458435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742100_1276 (size=12151) 2024-11-18T07:53:18,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916458638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916458648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916458648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916458658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916458658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/29928505f10747fdbe7dfd048f6fd8cd 2024-11-18T07:53:18,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/6b07c59620e04239bf80e356426155d7 is 50, key is test_row_0/C:col10/1731916397060/Put/seqid=0 2024-11-18T07:53:18,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916458948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916458958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916458969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916458969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742101_1277 (size=12151) 2024-11-18T07:53:18,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/6b07c59620e04239bf80e356426155d7 2024-11-18T07:53:18,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916458978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:18,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/953765d871134cbf902decd45b3cdfc2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/953765d871134cbf902decd45b3cdfc2 2024-11-18T07:53:19,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/953765d871134cbf902decd45b3cdfc2, entries=200, sequenceid=231, filesize=14.2 K 2024-11-18T07:53:19,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/29928505f10747fdbe7dfd048f6fd8cd as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/29928505f10747fdbe7dfd048f6fd8cd 2024-11-18T07:53:19,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/29928505f10747fdbe7dfd048f6fd8cd, entries=150, sequenceid=231, filesize=11.9 K 2024-11-18T07:53:19,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/6b07c59620e04239bf80e356426155d7 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/6b07c59620e04239bf80e356426155d7 2024-11-18T07:53:19,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/6b07c59620e04239bf80e356426155d7, entries=150, sequenceid=231, filesize=11.9 K 2024-11-18T07:53:19,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 1e0c954a4c9c2df78d79f9a8a402493e in 808ms, sequenceid=231, compaction requested=true 2024-11-18T07:53:19,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:19,048 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:53:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:53:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:53:19,049 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:19,050 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:19,050 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/B is initiating minor compaction (all files) 2024-11-18T07:53:19,050 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/B in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:19,050 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/23044d9376c048178673f8091128b639, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/71a9fe6dcb1e4303b35f3cc77468122f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/29928505f10747fdbe7dfd048f6fd8cd] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=36.0 K 2024-11-18T07:53:19,051 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:19,051 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/A is initiating minor compaction (all files) 2024-11-18T07:53:19,051 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/A in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:19,051 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/4664e164f1714af6a85a085774e00018, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/15ee1f9e21e748b6ae558c985eb39cd2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/953765d871134cbf902decd45b3cdfc2] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=38.4 K 2024-11-18T07:53:19,052 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 23044d9376c048178673f8091128b639, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731916395793 2024-11-18T07:53:19,052 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4664e164f1714af6a85a085774e00018, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731916395793 2024-11-18T07:53:19,054 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15ee1f9e21e748b6ae558c985eb39cd2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731916395835 2024-11-18T07:53:19,054 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 71a9fe6dcb1e4303b35f3cc77468122f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731916395835 2024-11-18T07:53:19,054 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 29928505f10747fdbe7dfd048f6fd8cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731916397052 2024-11-18T07:53:19,055 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 953765d871134cbf902decd45b3cdfc2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731916397052 2024-11-18T07:53:19,065 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#B#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:19,065 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/ae5a99f617d3419ea879064cb519bcb3 is 50, key is test_row_0/B:col10/1731916397060/Put/seqid=0 2024-11-18T07:53:19,073 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#A#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:19,074 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/af83d863e7b54a34a11cb880573d37e2 is 50, key is test_row_0/A:col10/1731916397060/Put/seqid=0 2024-11-18T07:53:19,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742102_1278 (size=12697) 2024-11-18T07:53:19,133 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/ae5a99f617d3419ea879064cb519bcb3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ae5a99f617d3419ea879064cb519bcb3 2024-11-18T07:53:19,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742103_1279 (size=12697) 2024-11-18T07:53:19,141 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/B of 1e0c954a4c9c2df78d79f9a8a402493e into ae5a99f617d3419ea879064cb519bcb3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:19,141 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:19,141 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/B, priority=13, startTime=1731916399049; duration=0sec 2024-11-18T07:53:19,141 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:19,141 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:B 2024-11-18T07:53:19,141 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:19,142 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:19,143 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/C is initiating minor compaction (all files) 2024-11-18T07:53:19,143 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/C in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:19,143 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0000dcafcc5045b6b3d34e8b7b79c445, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/64d5706650b3426ab4aa9f720ceb0ee0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/6b07c59620e04239bf80e356426155d7] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=36.0 K 2024-11-18T07:53:19,144 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0000dcafcc5045b6b3d34e8b7b79c445, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731916395793 2024-11-18T07:53:19,144 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 64d5706650b3426ab4aa9f720ceb0ee0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731916395835 2024-11-18T07:53:19,145 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b07c59620e04239bf80e356426155d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731916397052 2024-11-18T07:53:19,205 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#C#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:19,206 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/34828c48d4ee4c64a6eed4ea67124f47 is 50, key is test_row_0/C:col10/1731916397060/Put/seqid=0 2024-11-18T07:53:19,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742104_1280 (size=12697) 2024-11-18T07:53:19,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T07:53:19,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:19,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:19,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:19,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:19,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:19,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:19,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/ec3f26a91e7141edbda74098fd558d6e is 50, key is test_row_0/A:col10/1731916398304/Put/seqid=0 2024-11-18T07:53:19,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742105_1281 (size=14541) 2024-11-18T07:53:19,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916459513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:19,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:19,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916459515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:19,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:19,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:19,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/ec3f26a91e7141edbda74098fd558d6e 2024-11-18T07:53:19,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916459519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:19,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916459523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:19,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916459523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:19,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/b0c24cf44701496da4e1cd8650d5ad8d is 50, key is test_row_0/B:col10/1731916398304/Put/seqid=0 2024-11-18T07:53:19,569 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/af83d863e7b54a34a11cb880573d37e2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/af83d863e7b54a34a11cb880573d37e2 2024-11-18T07:53:19,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742106_1282 (size=12151) 2024-11-18T07:53:19,602 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/A of 1e0c954a4c9c2df78d79f9a8a402493e into af83d863e7b54a34a11cb880573d37e2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:19,603 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:19,603 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/A, priority=13, startTime=1731916399048; duration=0sec 2024-11-18T07:53:19,603 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:19,603 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:A 2024-11-18T07:53:19,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:19,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916459629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:19,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:19,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916459629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:19,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:19,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916459638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:19,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916459638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:19,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916459638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:19,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:19,679 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/34828c48d4ee4c64a6eed4ea67124f47 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/34828c48d4ee4c64a6eed4ea67124f47 2024-11-18T07:53:19,686 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/C of 1e0c954a4c9c2df78d79f9a8a402493e into 34828c48d4ee4c64a6eed4ea67124f47(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:19,686 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:19,687 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/C, priority=13, startTime=1731916399049; duration=0sec 2024-11-18T07:53:19,687 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:19,687 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:C 2024-11-18T07:53:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T07:53:19,798 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:19,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-18T07:53:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T07:53:19,807 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:19,811 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:19,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:19,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916459845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:19,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:19,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916459845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:19,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:19,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916459848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:19,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:19,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916459848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:19,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:19,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:19,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916459859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:19,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T07:53:19,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T07:53:19,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:19,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:19,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:19,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:19,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:20,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/b0c24cf44701496da4e1cd8650d5ad8d 2024-11-18T07:53:20,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/72c987a33d394938824ded655e75ace1 is 50, key is test_row_0/C:col10/1731916398304/Put/seqid=0 2024-11-18T07:53:20,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742107_1283 (size=12151) 2024-11-18T07:53:20,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T07:53:20,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T07:53:20,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:20,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:20,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:20,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:20,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:20,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:20,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/72c987a33d394938824ded655e75ace1 2024-11-18T07:53:20,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/ec3f26a91e7141edbda74098fd558d6e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/ec3f26a91e7141edbda74098fd558d6e 2024-11-18T07:53:20,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916460158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/ec3f26a91e7141edbda74098fd558d6e, entries=200, sequenceid=247, filesize=14.2 K 2024-11-18T07:53:20,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/b0c24cf44701496da4e1cd8650d5ad8d as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/b0c24cf44701496da4e1cd8650d5ad8d 2024-11-18T07:53:20,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/b0c24cf44701496da4e1cd8650d5ad8d, entries=150, sequenceid=247, filesize=11.9 K 2024-11-18T07:53:20,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/72c987a33d394938824ded655e75ace1 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/72c987a33d394938824ded655e75ace1 2024-11-18T07:53:20,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916460168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/72c987a33d394938824ded655e75ace1, entries=150, sequenceid=247, filesize=11.9 K 2024-11-18T07:53:20,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1e0c954a4c9c2df78d79f9a8a402493e in 707ms, sequenceid=247, compaction requested=false 2024-11-18T07:53:20,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:20,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T07:53:20,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:20,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:20,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:20,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:20,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:20,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:20,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/d021e1cf4bac412086dc54a6f3e9191c is 50, key is test_row_0/A:col10/1731916399520/Put/seqid=0 2024-11-18T07:53:20,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742108_1284 (size=14741) 2024-11-18T07:53:20,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/d021e1cf4bac412086dc54a6f3e9191c 2024-11-18T07:53:20,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916460241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916460242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/eb17abb4a0db48e4b0c2d3286a71c771 is 50, key is test_row_0/B:col10/1731916399520/Put/seqid=0 2024-11-18T07:53:20,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916460255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T07:53:20,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:20,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:20,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:20,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:20,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:20,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742109_1285 (size=12301) 2024-11-18T07:53:20,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/eb17abb4a0db48e4b0c2d3286a71c771 2024-11-18T07:53:20,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/221be84108f046e9a34d0ba4a6b8424a is 50, key is test_row_0/C:col10/1731916399520/Put/seqid=0 2024-11-18T07:53:20,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916460372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916460372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742110_1286 (size=12301) 2024-11-18T07:53:20,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/221be84108f046e9a34d0ba4a6b8424a 2024-11-18T07:53:20,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/d021e1cf4bac412086dc54a6f3e9191c as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/d021e1cf4bac412086dc54a6f3e9191c 2024-11-18T07:53:20,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916460387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/d021e1cf4bac412086dc54a6f3e9191c, entries=200, sequenceid=267, filesize=14.4 K 2024-11-18T07:53:20,405 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-18T07:53:20,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/eb17abb4a0db48e4b0c2d3286a71c771 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/eb17abb4a0db48e4b0c2d3286a71c771 2024-11-18T07:53:20,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/eb17abb4a0db48e4b0c2d3286a71c771, entries=150, sequenceid=267, filesize=12.0 K 2024-11-18T07:53:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T07:53:20,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/221be84108f046e9a34d0ba4a6b8424a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/221be84108f046e9a34d0ba4a6b8424a 2024-11-18T07:53:20,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T07:53:20,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/221be84108f046e9a34d0ba4a6b8424a, entries=150, sequenceid=267, filesize=12.0 K 2024-11-18T07:53:20,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:20,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:20,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:20,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:20,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:20,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1e0c954a4c9c2df78d79f9a8a402493e in 277ms, sequenceid=267, compaction requested=true 2024-11-18T07:53:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:53:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:20,462 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:53:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:53:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:53:20,462 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:20,464 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:20,464 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:20,464 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/A is initiating minor compaction (all files) 2024-11-18T07:53:20,464 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/B is initiating minor compaction (all files) 2024-11-18T07:53:20,464 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/A in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:20,464 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/B in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:20,464 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/af83d863e7b54a34a11cb880573d37e2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/ec3f26a91e7141edbda74098fd558d6e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/d021e1cf4bac412086dc54a6f3e9191c] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=41.0 K 2024-11-18T07:53:20,464 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ae5a99f617d3419ea879064cb519bcb3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/b0c24cf44701496da4e1cd8650d5ad8d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/eb17abb4a0db48e4b0c2d3286a71c771] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=36.3 K 2024-11-18T07:53:20,465 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae5a99f617d3419ea879064cb519bcb3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731916397052 2024-11-18T07:53:20,465 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting af83d863e7b54a34a11cb880573d37e2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731916397052 2024-11-18T07:53:20,465 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting ec3f26a91e7141edbda74098fd558d6e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731916398292 2024-11-18T07:53:20,465 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0c24cf44701496da4e1cd8650d5ad8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731916398304 2024-11-18T07:53:20,466 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting d021e1cf4bac412086dc54a6f3e9191c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731916399515 2024-11-18T07:53:20,466 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb17abb4a0db48e4b0c2d3286a71c771, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731916399519 2024-11-18T07:53:20,488 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#B#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:20,489 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/da28050b46d54fde89d181e79a04541f is 50, key is test_row_0/B:col10/1731916399520/Put/seqid=0 2024-11-18T07:53:20,490 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#A#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:20,490 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/1c7dfee77ea14723b922c261555e57c5 is 50, key is test_row_0/A:col10/1731916399520/Put/seqid=0 2024-11-18T07:53:20,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742112_1288 (size=12949) 2024-11-18T07:53:20,559 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/1c7dfee77ea14723b922c261555e57c5 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/1c7dfee77ea14723b922c261555e57c5 2024-11-18T07:53:20,579 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/A of 1e0c954a4c9c2df78d79f9a8a402493e into 1c7dfee77ea14723b922c261555e57c5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:20,579 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:20,579 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/A, priority=13, startTime=1731916400462; duration=0sec 2024-11-18T07:53:20,579 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:20,579 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:A 2024-11-18T07:53:20,580 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:20,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742111_1287 (size=12949) 2024-11-18T07:53:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:20,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T07:53:20,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:20,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:20,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:20,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:20,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:20,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:20,592 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:20,592 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/C is initiating minor compaction (all files) 2024-11-18T07:53:20,592 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/C in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:20,592 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/34828c48d4ee4c64a6eed4ea67124f47, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/72c987a33d394938824ded655e75ace1, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/221be84108f046e9a34d0ba4a6b8424a] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=36.3 K 2024-11-18T07:53:20,593 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 34828c48d4ee4c64a6eed4ea67124f47, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731916397052 2024-11-18T07:53:20,594 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 72c987a33d394938824ded655e75ace1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731916398304 2024-11-18T07:53:20,596 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 221be84108f046e9a34d0ba4a6b8424a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731916399519 2024-11-18T07:53:20,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/6ea353d92ac04c938b08b87b8449609e is 50, key is test_row_0/A:col10/1731916400585/Put/seqid=0 2024-11-18T07:53:20,607 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/da28050b46d54fde89d181e79a04541f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/da28050b46d54fde89d181e79a04541f 2024-11-18T07:53:20,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T07:53:20,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:20,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:20,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:20,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:20,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:20,627 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/B of 1e0c954a4c9c2df78d79f9a8a402493e into da28050b46d54fde89d181e79a04541f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:20,627 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:20,627 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/B, priority=13, startTime=1731916400462; duration=0sec 2024-11-18T07:53:20,627 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:20,627 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:B 2024-11-18T07:53:20,637 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#C#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:20,637 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/0a6391498b5a473fad8e17df251a2d83 is 50, key is test_row_0/C:col10/1731916399520/Put/seqid=0 2024-11-18T07:53:20,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742113_1289 (size=14741) 2024-11-18T07:53:20,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/6ea353d92ac04c938b08b87b8449609e 2024-11-18T07:53:20,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/d328127a7d3a413bab18f9bf95b9aa63 is 50, key is test_row_0/B:col10/1731916400585/Put/seqid=0 2024-11-18T07:53:20,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742114_1290 (size=12949) 2024-11-18T07:53:20,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742115_1291 (size=12301) 2024-11-18T07:53:20,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/d328127a7d3a413bab18f9bf95b9aa63 2024-11-18T07:53:20,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916460691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916460693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916460696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916460717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916460723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/2a1cce45c02a48008c36d2d290c320c1 is 50, key is test_row_0/C:col10/1731916400585/Put/seqid=0 2024-11-18T07:53:20,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T07:53:20,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:20,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:20,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:20,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:20,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:20,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:20,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742116_1292 (size=12301) 2024-11-18T07:53:20,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/2a1cce45c02a48008c36d2d290c320c1 2024-11-18T07:53:20,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916460828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916460828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/6ea353d92ac04c938b08b87b8449609e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/6ea353d92ac04c938b08b87b8449609e 2024-11-18T07:53:20,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916460839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916460848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:20,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916460849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:20,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/6ea353d92ac04c938b08b87b8449609e, entries=200, sequenceid=283, filesize=14.4 K 2024-11-18T07:53:20,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/d328127a7d3a413bab18f9bf95b9aa63 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/d328127a7d3a413bab18f9bf95b9aa63 2024-11-18T07:53:20,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/d328127a7d3a413bab18f9bf95b9aa63, entries=150, sequenceid=283, filesize=12.0 K 2024-11-18T07:53:20,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/2a1cce45c02a48008c36d2d290c320c1 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/2a1cce45c02a48008c36d2d290c320c1 2024-11-18T07:53:20,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/2a1cce45c02a48008c36d2d290c320c1, entries=150, sequenceid=283, filesize=12.0 K 2024-11-18T07:53:20,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1e0c954a4c9c2df78d79f9a8a402493e in 300ms, sequenceid=283, compaction requested=false 2024-11-18T07:53:20,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:20,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T07:53:20,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:20,951 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T07:53:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T07:53:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:20,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/79a65ab0fcad48988d425b601b2cf557 is 50, key is test_row_0/A:col10/1731916400703/Put/seqid=0 2024-11-18T07:53:21,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742117_1293 (size=12301) 2024-11-18T07:53:21,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:21,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916461079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916461081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916461085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916461086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916461088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,139 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/0a6391498b5a473fad8e17df251a2d83 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0a6391498b5a473fad8e17df251a2d83 2024-11-18T07:53:21,148 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/C of 1e0c954a4c9c2df78d79f9a8a402493e into 0a6391498b5a473fad8e17df251a2d83(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:21,148 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:21,148 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/C, priority=13, startTime=1731916400462; duration=0sec 2024-11-18T07:53:21,148 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:21,148 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:C 2024-11-18T07:53:21,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916461198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916461203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916461209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916461209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916461209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916461411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916461421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,438 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/79a65ab0fcad48988d425b601b2cf557 2024-11-18T07:53:21,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916461430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916461430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916461431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/344fcdb68c424048b883a1b69364c023 is 50, key is test_row_0/B:col10/1731916400703/Put/seqid=0 2024-11-18T07:53:21,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742118_1294 (size=12301) 2024-11-18T07:53:21,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916461728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916461748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916461748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916461749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916461758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:21,914 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/344fcdb68c424048b883a1b69364c023 2024-11-18T07:53:21,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/8996e92121ea47c38cfe42992d07b2a3 is 50, key is test_row_0/C:col10/1731916400703/Put/seqid=0 2024-11-18T07:53:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T07:53:22,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742119_1295 (size=12301) 2024-11-18T07:53:22,038 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/8996e92121ea47c38cfe42992d07b2a3 2024-11-18T07:53:22,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/79a65ab0fcad48988d425b601b2cf557 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/79a65ab0fcad48988d425b601b2cf557 2024-11-18T07:53:22,066 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/79a65ab0fcad48988d425b601b2cf557, entries=150, sequenceid=301, filesize=12.0 K 2024-11-18T07:53:22,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/344fcdb68c424048b883a1b69364c023 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/344fcdb68c424048b883a1b69364c023 2024-11-18T07:53:22,080 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/344fcdb68c424048b883a1b69364c023, entries=150, sequenceid=301, filesize=12.0 K 2024-11-18T07:53:22,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/8996e92121ea47c38cfe42992d07b2a3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/8996e92121ea47c38cfe42992d07b2a3 2024-11-18T07:53:22,089 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/8996e92121ea47c38cfe42992d07b2a3, entries=150, sequenceid=301, filesize=12.0 K 2024-11-18T07:53:22,091 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1e0c954a4c9c2df78d79f9a8a402493e in 1140ms, sequenceid=301, compaction requested=true 2024-11-18T07:53:22,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:22,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:22,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-18T07:53:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-18T07:53:22,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-18T07:53:22,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2820 sec 2024-11-18T07:53:22,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.2950 sec 2024-11-18T07:53:22,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T07:53:22,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:22,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:22,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:22,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:22,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:22,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:22,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/9b20f7ff468f4852b069397cb3f44357 is 50, key is test_row_0/A:col10/1731916401081/Put/seqid=0 2024-11-18T07:53:22,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742120_1296 (size=14741) 2024-11-18T07:53:22,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916462340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:22,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916462343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:22,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916462355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:22,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916462361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:22,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916462360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:22,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:22,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916462469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:22,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:22,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916462469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916462470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:22,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:22,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916462479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:22,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:22,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916462488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:22,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916462691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:22,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916462695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916462695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:22,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916462703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:22,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:22,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916462709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:22,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:22,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:22,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/9b20f7ff468f4852b069397cb3f44357 2024-11-18T07:53:22,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/572177cf32204c09985f77ac36c5b18a is 50, key is test_row_0/B:col10/1731916401081/Put/seqid=0 2024-11-18T07:53:22,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742121_1297 (size=12301) 2024-11-18T07:53:23,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916463023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:23,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916463024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:23,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916463024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:23,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916463035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:23,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916463035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:23,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/572177cf32204c09985f77ac36c5b18a 2024-11-18T07:53:23,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/f0a58c95e7704226b5b32fee22b3e159 is 50, key is test_row_0/C:col10/1731916401081/Put/seqid=0 2024-11-18T07:53:23,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742122_1298 (size=12301) 2024-11-18T07:53:23,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/f0a58c95e7704226b5b32fee22b3e159 2024-11-18T07:53:23,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/9b20f7ff468f4852b069397cb3f44357 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/9b20f7ff468f4852b069397cb3f44357 2024-11-18T07:53:23,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/9b20f7ff468f4852b069397cb3f44357, entries=200, sequenceid=318, filesize=14.4 K 2024-11-18T07:53:23,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/572177cf32204c09985f77ac36c5b18a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/572177cf32204c09985f77ac36c5b18a 2024-11-18T07:53:23,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916463538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:23,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:23,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/572177cf32204c09985f77ac36c5b18a, entries=150, sequenceid=318, filesize=12.0 K 2024-11-18T07:53:23,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916463551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:23,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/f0a58c95e7704226b5b32fee22b3e159 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/f0a58c95e7704226b5b32fee22b3e159 2024-11-18T07:53:23,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916463551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:23,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:23,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:23,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916463558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916463572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:23,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:23,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:23,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/f0a58c95e7704226b5b32fee22b3e159, entries=150, sequenceid=318, filesize=12.0 K 2024-11-18T07:53:23,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 1e0c954a4c9c2df78d79f9a8a402493e in 1381ms, sequenceid=318, compaction requested=true 2024-11-18T07:53:23,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:23,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:53:23,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:23,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:53:23,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:53:23,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:53:23,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:53:23,638 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:53:23,638 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:53:23,655 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54732 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:53:23,655 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/A is initiating minor compaction (all files) 2024-11-18T07:53:23,655 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/A in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:23,655 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/1c7dfee77ea14723b922c261555e57c5, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/6ea353d92ac04c938b08b87b8449609e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/79a65ab0fcad48988d425b601b2cf557, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/9b20f7ff468f4852b069397cb3f44357] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=53.4 K 2024-11-18T07:53:23,655 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:53:23,656 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/B is initiating minor compaction (all files) 2024-11-18T07:53:23,656 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/B in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:23,656 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/da28050b46d54fde89d181e79a04541f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/d328127a7d3a413bab18f9bf95b9aa63, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/344fcdb68c424048b883a1b69364c023, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/572177cf32204c09985f77ac36c5b18a] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=48.7 K 2024-11-18T07:53:23,659 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c7dfee77ea14723b922c261555e57c5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731916399519 2024-11-18T07:53:23,659 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting da28050b46d54fde89d181e79a04541f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731916399519 2024-11-18T07:53:23,663 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting d328127a7d3a413bab18f9bf95b9aa63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731916400240 2024-11-18T07:53:23,663 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ea353d92ac04c938b08b87b8449609e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731916400228 2024-11-18T07:53:23,663 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79a65ab0fcad48988d425b601b2cf557, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731916400692 2024-11-18T07:53:23,663 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 344fcdb68c424048b883a1b69364c023, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731916400692 2024-11-18T07:53:23,671 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b20f7ff468f4852b069397cb3f44357, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731916401078 2024-11-18T07:53:23,671 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 572177cf32204c09985f77ac36c5b18a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731916401078 2024-11-18T07:53:23,726 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#B#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:23,727 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#A#compaction#256 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:23,727 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/4cea19b73b3d4551805660fc5ee427b5 is 50, key is test_row_0/A:col10/1731916401081/Put/seqid=0 2024-11-18T07:53:23,726 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/ed1ec5b002014b8c9c6e975215548244 is 50, key is test_row_0/B:col10/1731916401081/Put/seqid=0 2024-11-18T07:53:23,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742123_1299 (size=13085) 2024-11-18T07:53:23,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742124_1300 (size=13085) 2024-11-18T07:53:23,859 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/ed1ec5b002014b8c9c6e975215548244 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ed1ec5b002014b8c9c6e975215548244 2024-11-18T07:53:23,872 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/B of 1e0c954a4c9c2df78d79f9a8a402493e into ed1ec5b002014b8c9c6e975215548244(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:23,872 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:23,872 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/B, priority=12, startTime=1731916403638; duration=0sec 2024-11-18T07:53:23,873 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:23,873 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:B 2024-11-18T07:53:23,873 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:53:23,880 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:53:23,880 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 1e0c954a4c9c2df78d79f9a8a402493e/C is initiating minor compaction (all files) 2024-11-18T07:53:23,880 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e0c954a4c9c2df78d79f9a8a402493e/C in TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:23,881 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0a6391498b5a473fad8e17df251a2d83, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/2a1cce45c02a48008c36d2d290c320c1, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/8996e92121ea47c38cfe42992d07b2a3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/f0a58c95e7704226b5b32fee22b3e159] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp, totalSize=48.7 K 2024-11-18T07:53:23,882 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a6391498b5a473fad8e17df251a2d83, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731916399519 2024-11-18T07:53:23,883 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a1cce45c02a48008c36d2d290c320c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731916400240 2024-11-18T07:53:23,883 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8996e92121ea47c38cfe42992d07b2a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731916400692 2024-11-18T07:53:23,884 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting f0a58c95e7704226b5b32fee22b3e159, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731916401078 2024-11-18T07:53:23,901 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e0c954a4c9c2df78d79f9a8a402493e#C#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:23,901 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/fca8a02c378545a687c01d50ddccba54 is 50, key is test_row_0/C:col10/1731916401081/Put/seqid=0 2024-11-18T07:53:23,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742125_1301 (size=13085) 2024-11-18T07:53:23,989 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/fca8a02c378545a687c01d50ddccba54 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/fca8a02c378545a687c01d50ddccba54 2024-11-18T07:53:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T07:53:24,000 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:24,001 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:24,008 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/C of 1e0c954a4c9c2df78d79f9a8a402493e into fca8a02c378545a687c01d50ddccba54(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:24,008 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:24,008 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/C, priority=12, startTime=1731916403638; duration=0sec 2024-11-18T07:53:24,008 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:24,008 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:C 2024-11-18T07:53:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-18T07:53:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T07:53:24,011 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:24,013 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:24,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T07:53:24,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-18T07:53:24,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:24,172 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T07:53:24,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:24,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:24,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:24,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:24,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:24,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/8c5e57dd3ca04267854cce70416d9dce is 50, key is test_row_0/A:col10/1731916402347/Put/seqid=0 2024-11-18T07:53:24,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742126_1302 (size=12301) 2024-11-18T07:53:24,246 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/4cea19b73b3d4551805660fc5ee427b5 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/4cea19b73b3d4551805660fc5ee427b5 2024-11-18T07:53:24,261 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e0c954a4c9c2df78d79f9a8a402493e/A of 1e0c954a4c9c2df78d79f9a8a402493e into 4cea19b73b3d4551805660fc5ee427b5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:24,261 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:24,261 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., storeName=1e0c954a4c9c2df78d79f9a8a402493e/A, priority=12, startTime=1731916403638; duration=0sec 2024-11-18T07:53:24,261 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:24,261 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:A 2024-11-18T07:53:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T07:53:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:24,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. as already flushing 2024-11-18T07:53:24,628 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/8c5e57dd3ca04267854cce70416d9dce 2024-11-18T07:53:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T07:53:24,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/d93c0c8663ed428fbf0056051017d789 is 50, key is test_row_0/B:col10/1731916402347/Put/seqid=0 2024-11-18T07:53:24,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916464663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:24,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40358 deadline: 1731916464663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:24,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916464669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:24,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:24,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:24,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:24,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731916464672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:24,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1731916464686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:24,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742127_1303 (size=12301) 2024-11-18T07:53:24,717 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/d93c0c8663ed428fbf0056051017d789 2024-11-18T07:53:24,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/ce795760156142f8b7f28f8a18a84bd4 is 50, key is test_row_0/C:col10/1731916402347/Put/seqid=0 2024-11-18T07:53:24,745 INFO [Thread-1092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1092. 2024-11-18T07:53:24,745 DEBUG [Thread-1092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:24,745 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:24,745 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:24,745 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:24,749 INFO [Thread-1086 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1086. 2024-11-18T07:53:24,749 DEBUG [Thread-1086 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:24,749 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:24,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:24,750 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:24,754 INFO [Thread-1094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1094. 2024-11-18T07:53:24,754 DEBUG [Thread-1094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:24,754 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:24,754 INFO [Thread-1090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1090. 2024-11-18T07:53:24,754 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:24,754 DEBUG [Thread-1090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:24,754 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:24,754 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:24,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:24,762 INFO [Thread-1088 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1088. 2024-11-18T07:53:24,762 DEBUG [Thread-1088 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:24,762 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:24,762 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:24,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:24,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:24,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742128_1304 (size=12301) 2024-11-18T07:53:24,764 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/ce795760156142f8b7f28f8a18a84bd4 2024-11-18T07:53:24,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/8c5e57dd3ca04267854cce70416d9dce as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/8c5e57dd3ca04267854cce70416d9dce 2024-11-18T07:53:24,778 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/8c5e57dd3ca04267854cce70416d9dce, entries=150, sequenceid=337, filesize=12.0 K 2024-11-18T07:53:24,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/d93c0c8663ed428fbf0056051017d789 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/d93c0c8663ed428fbf0056051017d789 2024-11-18T07:53:24,786 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/d93c0c8663ed428fbf0056051017d789, entries=150, sequenceid=337, filesize=12.0 K 2024-11-18T07:53:24,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/ce795760156142f8b7f28f8a18a84bd4 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/ce795760156142f8b7f28f8a18a84bd4 2024-11-18T07:53:24,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40350 deadline: 1731916464788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:24,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:24,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:24,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:24,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40346 deadline: 1731916464788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:24,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:24,794 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/ce795760156142f8b7f28f8a18a84bd4, entries=150, sequenceid=337, filesize=12.0 K 2024-11-18T07:53:24,795 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 1e0c954a4c9c2df78d79f9a8a402493e in 623ms, sequenceid=337, compaction requested=false 2024-11-18T07:53:24,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:24,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:24,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-18T07:53:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-18T07:53:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:24,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T07:53:24,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:24,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:24,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:24,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:24,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:24,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:24,802 INFO [Thread-1083 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1083. 2024-11-18T07:53:24,802 DEBUG [Thread-1083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:24,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-18T07:53:24,802 DEBUG [Thread-1083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:24,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 784 msec 2024-11-18T07:53:24,802 DEBUG [Thread-1083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:24,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:24,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 803 msec 2024-11-18T07:53:24,807 INFO [Thread-1081 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1081. 2024-11-18T07:53:24,807 DEBUG [Thread-1081 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:24,807 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:24,807 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:24,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:24,815 INFO [Thread-1079 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1079. 2024-11-18T07:53:24,815 DEBUG [Thread-1079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:24,815 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:24,816 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:24,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:24,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/5b4ca57974934ef09d45b99c6dbb5f02 is 50, key is test_row_0/A:col10/1731916404662/Put/seqid=0 2024-11-18T07:53:24,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742129_1305 (size=12301) 2024-11-18T07:53:25,013 INFO [Thread-1075 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1075. 2024-11-18T07:53:25,013 DEBUG [Thread-1075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:25,013 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:25,014 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:25,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:25,015 INFO [Thread-1077 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1077. 2024-11-18T07:53:25,015 DEBUG [Thread-1077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:25,015 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:25,015 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:25,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:25,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T07:53:25,159 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:25,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T07:53:25,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-18T07:53:25,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-18T07:53:25,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-18T07:53:25,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-18T07:53:25,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-18T07:53:25,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T07:53:25,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1283 2024-11-18T07:53:25,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1281 2024-11-18T07:53:25,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1302 2024-11-18T07:53:25,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1277 2024-11-18T07:53:25,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1274 2024-11-18T07:53:25,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T07:53:25,159 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T07:53:25,160 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T07:53:25,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:25,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:25,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T07:53:25,162 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:25,168 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916405168"}]},"ts":"1731916405168"} 2024-11-18T07:53:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T07:53:25,173 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T07:53:25,174 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T07:53:25,175 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T07:53:25,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e0c954a4c9c2df78d79f9a8a402493e, UNASSIGN}] 2024-11-18T07:53:25,179 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e0c954a4c9c2df78d79f9a8a402493e, UNASSIGN 2024-11-18T07:53:25,180 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=1e0c954a4c9c2df78d79f9a8a402493e, regionState=CLOSING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:53:25,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e0c954a4c9c2df78d79f9a8a402493e, UNASSIGN because future has completed 2024-11-18T07:53:25,183 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T07:53:25,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:53:25,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/5b4ca57974934ef09d45b99c6dbb5f02 2024-11-18T07:53:25,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T07:53:25,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/153343212a114dc8804b2313d95ece27 is 50, key is test_row_0/B:col10/1731916404662/Put/seqid=0 2024-11-18T07:53:25,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742130_1306 (size=12301) 2024-11-18T07:53:25,338 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(122): Close 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:25,338 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T07:53:25,338 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1722): Closing 1e0c954a4c9c2df78d79f9a8a402493e, disabling compactions & flushes 2024-11-18T07:53:25,338 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T07:53:25,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/153343212a114dc8804b2313d95ece27 2024-11-18T07:53:25,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/853fd5978f724570917563afb471b59e is 50, key is test_row_0/C:col10/1731916404662/Put/seqid=0 2024-11-18T07:53:25,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742131_1307 (size=12301) 2024-11-18T07:53:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T07:53:26,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/853fd5978f724570917563afb471b59e 2024-11-18T07:53:26,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/5b4ca57974934ef09d45b99c6dbb5f02 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/5b4ca57974934ef09d45b99c6dbb5f02 2024-11-18T07:53:26,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/5b4ca57974934ef09d45b99c6dbb5f02, entries=150, sequenceid=355, filesize=12.0 K 2024-11-18T07:53:26,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/153343212a114dc8804b2313d95ece27 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/153343212a114dc8804b2313d95ece27 2024-11-18T07:53:26,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/153343212a114dc8804b2313d95ece27, entries=150, sequenceid=355, filesize=12.0 K 2024-11-18T07:53:26,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/853fd5978f724570917563afb471b59e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/853fd5978f724570917563afb471b59e 2024-11-18T07:53:26,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/853fd5978f724570917563afb471b59e, entries=150, sequenceid=355, filesize=12.0 K 2024-11-18T07:53:26,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=26.84 KB/27480 for 1e0c954a4c9c2df78d79f9a8a402493e in 1442ms, sequenceid=355, compaction requested=true 2024-11-18T07:53:26,243 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:26,243 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:26,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e0c954a4c9c2df78d79f9a8a402493e: 2024-11-18T07:53:26,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:53:26,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:26,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:53:26,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:53:26,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e0c954a4c9c2df78d79f9a8a402493e:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:53:26,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T07:53:26,244 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. after waiting 1 ms 2024-11-18T07:53:26,244 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:26,244 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(2902): Flushing 1e0c954a4c9c2df78d79f9a8a402493e 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-18T07:53:26,244 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. because compaction request was cancelled 2024-11-18T07:53:26,244 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:A 2024-11-18T07:53:26,244 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. because compaction request was cancelled 2024-11-18T07:53:26,244 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:C 2024-11-18T07:53:26,244 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. because compaction request was cancelled 2024-11-18T07:53:26,244 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e0c954a4c9c2df78d79f9a8a402493e:B 2024-11-18T07:53:26,244 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=A 2024-11-18T07:53:26,245 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:26,245 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=B 2024-11-18T07:53:26,245 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:26,245 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1e0c954a4c9c2df78d79f9a8a402493e, store=C 2024-11-18T07:53:26,245 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:26,265 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/fa83f96e04d548bfa59e6c0b4000c9fd is 50, key is test_row_0/A:col10/1731916405001/Put/seqid=0 2024-11-18T07:53:26,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742132_1308 (size=12301) 2024-11-18T07:53:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T07:53:26,703 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/fa83f96e04d548bfa59e6c0b4000c9fd 2024-11-18T07:53:26,726 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/ff86e113806a47c8be0b70a4f189e5fa is 50, key is test_row_0/B:col10/1731916405001/Put/seqid=0 2024-11-18T07:53:26,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742133_1309 (size=12301) 2024-11-18T07:53:27,159 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/ff86e113806a47c8be0b70a4f189e5fa 2024-11-18T07:53:27,188 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/0b802731de15484fa9827dff00d2b142 is 50, key is test_row_0/C:col10/1731916405001/Put/seqid=0 2024-11-18T07:53:27,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742134_1310 (size=12301) 2024-11-18T07:53:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T07:53:27,648 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/0b802731de15484fa9827dff00d2b142 2024-11-18T07:53:27,653 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/A/fa83f96e04d548bfa59e6c0b4000c9fd as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/fa83f96e04d548bfa59e6c0b4000c9fd 2024-11-18T07:53:27,657 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/fa83f96e04d548bfa59e6c0b4000c9fd, entries=150, sequenceid=362, filesize=12.0 K 2024-11-18T07:53:27,658 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/B/ff86e113806a47c8be0b70a4f189e5fa as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ff86e113806a47c8be0b70a4f189e5fa 2024-11-18T07:53:27,663 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ff86e113806a47c8be0b70a4f189e5fa, entries=150, sequenceid=362, filesize=12.0 K 2024-11-18T07:53:27,664 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/.tmp/C/0b802731de15484fa9827dff00d2b142 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0b802731de15484fa9827dff00d2b142 2024-11-18T07:53:27,668 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0b802731de15484fa9827dff00d2b142, entries=150, sequenceid=362, filesize=12.0 K 2024-11-18T07:53:27,669 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1e0c954a4c9c2df78d79f9a8a402493e in 1425ms, sequenceid=362, compaction requested=true 2024-11-18T07:53:27,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/c5d8a4d9478a4fc1977a8407330a650d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/f52abba34ce943f89bcd1d6f14ee4435, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/c335ea01db874b07aef640a153661c8a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/e8d12613070d420eb520f74773486a44, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/bc205bbfe5f849438d15b77c3a98d083, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/06fe1e4e2271476293b8a2f2de80b835, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/929eafc549fe4436b6f35a1f49dad675, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/36e2de44a84b4c539d8435846198fb14, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/6c01dd4abed84e63a914195f6c1f9aab, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/a3bee9ca8f4b44f4938f9d16f8f1fb2b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/8f3b62e6ebdf41e3861cac6b44d526f7, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/917fd4f86cba4ac0834af5d3d5ee4431, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/4664e164f1714af6a85a085774e00018, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/af74f5649e0c41d583905eaf81fd56f3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/15ee1f9e21e748b6ae558c985eb39cd2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/953765d871134cbf902decd45b3cdfc2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/af83d863e7b54a34a11cb880573d37e2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/ec3f26a91e7141edbda74098fd558d6e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/d021e1cf4bac412086dc54a6f3e9191c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/1c7dfee77ea14723b922c261555e57c5, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/6ea353d92ac04c938b08b87b8449609e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/79a65ab0fcad48988d425b601b2cf557, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/9b20f7ff468f4852b069397cb3f44357] to archive 2024-11-18T07:53:27,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:53:27,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/c5d8a4d9478a4fc1977a8407330a650d to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/c5d8a4d9478a4fc1977a8407330a650d 2024-11-18T07:53:27,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/f52abba34ce943f89bcd1d6f14ee4435 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/f52abba34ce943f89bcd1d6f14ee4435 2024-11-18T07:53:27,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/c335ea01db874b07aef640a153661c8a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/c335ea01db874b07aef640a153661c8a 2024-11-18T07:53:27,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/e8d12613070d420eb520f74773486a44 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/e8d12613070d420eb520f74773486a44 2024-11-18T07:53:27,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/bc205bbfe5f849438d15b77c3a98d083 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/bc205bbfe5f849438d15b77c3a98d083 2024-11-18T07:53:27,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/06fe1e4e2271476293b8a2f2de80b835 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/06fe1e4e2271476293b8a2f2de80b835 2024-11-18T07:53:27,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/929eafc549fe4436b6f35a1f49dad675 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/929eafc549fe4436b6f35a1f49dad675 2024-11-18T07:53:27,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/36e2de44a84b4c539d8435846198fb14 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/36e2de44a84b4c539d8435846198fb14 2024-11-18T07:53:27,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/6c01dd4abed84e63a914195f6c1f9aab to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/6c01dd4abed84e63a914195f6c1f9aab 2024-11-18T07:53:27,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/a3bee9ca8f4b44f4938f9d16f8f1fb2b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/a3bee9ca8f4b44f4938f9d16f8f1fb2b 2024-11-18T07:53:27,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/8f3b62e6ebdf41e3861cac6b44d526f7 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/8f3b62e6ebdf41e3861cac6b44d526f7 2024-11-18T07:53:27,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/917fd4f86cba4ac0834af5d3d5ee4431 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/917fd4f86cba4ac0834af5d3d5ee4431 2024-11-18T07:53:27,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/4664e164f1714af6a85a085774e00018 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/4664e164f1714af6a85a085774e00018 2024-11-18T07:53:27,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/af74f5649e0c41d583905eaf81fd56f3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/af74f5649e0c41d583905eaf81fd56f3 2024-11-18T07:53:27,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/15ee1f9e21e748b6ae558c985eb39cd2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/15ee1f9e21e748b6ae558c985eb39cd2 2024-11-18T07:53:27,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/953765d871134cbf902decd45b3cdfc2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/953765d871134cbf902decd45b3cdfc2 2024-11-18T07:53:27,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/af83d863e7b54a34a11cb880573d37e2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/af83d863e7b54a34a11cb880573d37e2 2024-11-18T07:53:27,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/ec3f26a91e7141edbda74098fd558d6e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/ec3f26a91e7141edbda74098fd558d6e 2024-11-18T07:53:27,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/d021e1cf4bac412086dc54a6f3e9191c to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/d021e1cf4bac412086dc54a6f3e9191c 2024-11-18T07:53:27,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/1c7dfee77ea14723b922c261555e57c5 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/1c7dfee77ea14723b922c261555e57c5 2024-11-18T07:53:27,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/6ea353d92ac04c938b08b87b8449609e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/6ea353d92ac04c938b08b87b8449609e 2024-11-18T07:53:27,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/79a65ab0fcad48988d425b601b2cf557 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/79a65ab0fcad48988d425b601b2cf557 2024-11-18T07:53:27,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/9b20f7ff468f4852b069397cb3f44357 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/9b20f7ff468f4852b069397cb3f44357 2024-11-18T07:53:27,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/4ac1bfb6e0274fc9904f9db01cf9517b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/7666878fdfff4f8fac362828872c4ecd, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/5728298c272e4131ab2aa18960a8373b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/af61d337779f42d5b67a2264272854e2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/eb19e409f1f447a1a68d3c5fafbc6d92, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ec5a735dce374bcf976cd1275e0f05f3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/e697fe8c853c448a84730488c26b9379, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/6fdb8b68257a46e6b9a38f73ae9f892f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/cbef7c0379f54bff93777153763127f2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/0d2c8db7186a469c98131adb3c365f63, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/0ca85685ef0047f989ddfe3c01fd6f12, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/f2f8a53bd8704492abea70a5699d4cf0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/23044d9376c048178673f8091128b639, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/07cb5a6839a5495e9ee62ef9cb5c2862, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/71a9fe6dcb1e4303b35f3cc77468122f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ae5a99f617d3419ea879064cb519bcb3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/29928505f10747fdbe7dfd048f6fd8cd, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/b0c24cf44701496da4e1cd8650d5ad8d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/da28050b46d54fde89d181e79a04541f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/eb17abb4a0db48e4b0c2d3286a71c771, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/d328127a7d3a413bab18f9bf95b9aa63, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/344fcdb68c424048b883a1b69364c023, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/572177cf32204c09985f77ac36c5b18a] to archive 2024-11-18T07:53:27,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:53:27,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/4ac1bfb6e0274fc9904f9db01cf9517b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/4ac1bfb6e0274fc9904f9db01cf9517b 2024-11-18T07:53:27,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/7666878fdfff4f8fac362828872c4ecd to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/7666878fdfff4f8fac362828872c4ecd 2024-11-18T07:53:27,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/5728298c272e4131ab2aa18960a8373b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/5728298c272e4131ab2aa18960a8373b 2024-11-18T07:53:27,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/af61d337779f42d5b67a2264272854e2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/af61d337779f42d5b67a2264272854e2 2024-11-18T07:53:27,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/eb19e409f1f447a1a68d3c5fafbc6d92 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/eb19e409f1f447a1a68d3c5fafbc6d92 2024-11-18T07:53:27,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ec5a735dce374bcf976cd1275e0f05f3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ec5a735dce374bcf976cd1275e0f05f3 2024-11-18T07:53:27,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/e697fe8c853c448a84730488c26b9379 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/e697fe8c853c448a84730488c26b9379 2024-11-18T07:53:27,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/6fdb8b68257a46e6b9a38f73ae9f892f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/6fdb8b68257a46e6b9a38f73ae9f892f 2024-11-18T07:53:27,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/cbef7c0379f54bff93777153763127f2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/cbef7c0379f54bff93777153763127f2 2024-11-18T07:53:27,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/0d2c8db7186a469c98131adb3c365f63 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/0d2c8db7186a469c98131adb3c365f63 2024-11-18T07:53:27,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/0ca85685ef0047f989ddfe3c01fd6f12 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/0ca85685ef0047f989ddfe3c01fd6f12 2024-11-18T07:53:27,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/f2f8a53bd8704492abea70a5699d4cf0 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/f2f8a53bd8704492abea70a5699d4cf0 2024-11-18T07:53:27,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/23044d9376c048178673f8091128b639 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/23044d9376c048178673f8091128b639 2024-11-18T07:53:27,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/07cb5a6839a5495e9ee62ef9cb5c2862 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/07cb5a6839a5495e9ee62ef9cb5c2862 2024-11-18T07:53:27,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/71a9fe6dcb1e4303b35f3cc77468122f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/71a9fe6dcb1e4303b35f3cc77468122f 2024-11-18T07:53:27,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ae5a99f617d3419ea879064cb519bcb3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ae5a99f617d3419ea879064cb519bcb3 2024-11-18T07:53:27,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/29928505f10747fdbe7dfd048f6fd8cd to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/29928505f10747fdbe7dfd048f6fd8cd 2024-11-18T07:53:27,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/b0c24cf44701496da4e1cd8650d5ad8d to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/b0c24cf44701496da4e1cd8650d5ad8d 2024-11-18T07:53:27,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/da28050b46d54fde89d181e79a04541f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/da28050b46d54fde89d181e79a04541f 2024-11-18T07:53:27,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/eb17abb4a0db48e4b0c2d3286a71c771 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/eb17abb4a0db48e4b0c2d3286a71c771 2024-11-18T07:53:27,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/d328127a7d3a413bab18f9bf95b9aa63 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/d328127a7d3a413bab18f9bf95b9aa63 2024-11-18T07:53:27,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/344fcdb68c424048b883a1b69364c023 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/344fcdb68c424048b883a1b69364c023 2024-11-18T07:53:27,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/572177cf32204c09985f77ac36c5b18a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/572177cf32204c09985f77ac36c5b18a 2024-11-18T07:53:27,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/3109ca67c3f14d109e2fff7a724c903b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/7969631957e047a4865b76c155ba362e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/2dc0dda9575745dbabd06bbe7f7e2aa2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/321ef2ce32eb4d1395dbcfc9c3a06110, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/86dcddb137214e918e086b042d895679, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5d851db5453c46a4886208a157a13e03, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/2a41b1f7fa20497c87409535393a868d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/883fd27cc96d48148e9f22b9cd266c84, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0ddf5320cb0547d9a3271ec55a5171c8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5293ad29fa9d49be94c7eb64d23a8f8d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5b6104d2c5f64f5cae175018db039a6f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/ac2c828add6a4df68f39c7b9ab24ae35, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0000dcafcc5045b6b3d34e8b7b79c445, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/9f31e1a58d0e42deaf97884ab04619c6, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/64d5706650b3426ab4aa9f720ceb0ee0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/34828c48d4ee4c64a6eed4ea67124f47, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/6b07c59620e04239bf80e356426155d7, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/72c987a33d394938824ded655e75ace1, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0a6391498b5a473fad8e17df251a2d83, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/221be84108f046e9a34d0ba4a6b8424a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/2a1cce45c02a48008c36d2d290c320c1, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/8996e92121ea47c38cfe42992d07b2a3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/f0a58c95e7704226b5b32fee22b3e159] to archive 2024-11-18T07:53:27,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:53:27,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/3109ca67c3f14d109e2fff7a724c903b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/3109ca67c3f14d109e2fff7a724c903b 2024-11-18T07:53:27,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/7969631957e047a4865b76c155ba362e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/7969631957e047a4865b76c155ba362e 2024-11-18T07:53:27,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/2dc0dda9575745dbabd06bbe7f7e2aa2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/2dc0dda9575745dbabd06bbe7f7e2aa2 2024-11-18T07:53:27,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/321ef2ce32eb4d1395dbcfc9c3a06110 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/321ef2ce32eb4d1395dbcfc9c3a06110 2024-11-18T07:53:27,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/86dcddb137214e918e086b042d895679 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/86dcddb137214e918e086b042d895679 2024-11-18T07:53:27,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5d851db5453c46a4886208a157a13e03 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5d851db5453c46a4886208a157a13e03 2024-11-18T07:53:27,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/2a41b1f7fa20497c87409535393a868d to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/2a41b1f7fa20497c87409535393a868d 2024-11-18T07:53:27,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/883fd27cc96d48148e9f22b9cd266c84 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/883fd27cc96d48148e9f22b9cd266c84 2024-11-18T07:53:27,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0ddf5320cb0547d9a3271ec55a5171c8 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0ddf5320cb0547d9a3271ec55a5171c8 2024-11-18T07:53:27,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5293ad29fa9d49be94c7eb64d23a8f8d to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5293ad29fa9d49be94c7eb64d23a8f8d 2024-11-18T07:53:27,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5b6104d2c5f64f5cae175018db039a6f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/5b6104d2c5f64f5cae175018db039a6f 2024-11-18T07:53:27,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/ac2c828add6a4df68f39c7b9ab24ae35 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/ac2c828add6a4df68f39c7b9ab24ae35 2024-11-18T07:53:27,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0000dcafcc5045b6b3d34e8b7b79c445 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0000dcafcc5045b6b3d34e8b7b79c445 2024-11-18T07:53:27,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/9f31e1a58d0e42deaf97884ab04619c6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/9f31e1a58d0e42deaf97884ab04619c6 2024-11-18T07:53:27,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/64d5706650b3426ab4aa9f720ceb0ee0 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/64d5706650b3426ab4aa9f720ceb0ee0 2024-11-18T07:53:27,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/34828c48d4ee4c64a6eed4ea67124f47 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/34828c48d4ee4c64a6eed4ea67124f47 2024-11-18T07:53:27,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/6b07c59620e04239bf80e356426155d7 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/6b07c59620e04239bf80e356426155d7 2024-11-18T07:53:27,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/72c987a33d394938824ded655e75ace1 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/72c987a33d394938824ded655e75ace1 2024-11-18T07:53:27,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0a6391498b5a473fad8e17df251a2d83 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0a6391498b5a473fad8e17df251a2d83 2024-11-18T07:53:27,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/221be84108f046e9a34d0ba4a6b8424a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/221be84108f046e9a34d0ba4a6b8424a 2024-11-18T07:53:27,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/2a1cce45c02a48008c36d2d290c320c1 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/2a1cce45c02a48008c36d2d290c320c1 2024-11-18T07:53:27,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/8996e92121ea47c38cfe42992d07b2a3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/8996e92121ea47c38cfe42992d07b2a3 2024-11-18T07:53:27,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/f0a58c95e7704226b5b32fee22b3e159 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/f0a58c95e7704226b5b32fee22b3e159 2024-11-18T07:53:27,756 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/recovered.edits/365.seqid, newMaxSeqId=365, maxSeqId=1 2024-11-18T07:53:27,757 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. 2024-11-18T07:53:27,757 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1676): Region close journal for 1e0c954a4c9c2df78d79f9a8a402493e: Waiting for close lock at 1731916405338Running coprocessor pre-close hooks at 1731916405338Disabling compacts and flushes for region at 1731916405338Disabling writes for close at 1731916406244 (+906 ms)Obtaining lock to block concurrent updates at 1731916406244Preparing flush snapshotting stores in 1e0c954a4c9c2df78d79f9a8a402493e at 1731916406244Finished memstore snapshotting TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731916406245 (+1 ms)Flushing stores of TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e. at 1731916406246 (+1 ms)Flushing 1e0c954a4c9c2df78d79f9a8a402493e/A: creating writer at 1731916406246Flushing 1e0c954a4c9c2df78d79f9a8a402493e/A: appending metadata at 1731916406265 (+19 ms)Flushing 1e0c954a4c9c2df78d79f9a8a402493e/A: closing flushed file at 1731916406265Flushing 1e0c954a4c9c2df78d79f9a8a402493e/B: creating writer at 1731916406711 (+446 ms)Flushing 1e0c954a4c9c2df78d79f9a8a402493e/B: appending metadata at 1731916406725 (+14 ms)Flushing 1e0c954a4c9c2df78d79f9a8a402493e/B: closing flushed file at 1731916406725Flushing 1e0c954a4c9c2df78d79f9a8a402493e/C: creating writer at 1731916407172 (+447 ms)Flushing 1e0c954a4c9c2df78d79f9a8a402493e/C: appending metadata at 1731916407187 (+15 ms)Flushing 1e0c954a4c9c2df78d79f9a8a402493e/C: closing flushed file at 1731916407187Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24729860: reopening flushed file at 1731916407652 (+465 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b3cb39d: reopening flushed file at 1731916407658 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44ca8766: reopening flushed file at 1731916407663 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1e0c954a4c9c2df78d79f9a8a402493e in 1425ms, sequenceid=362, compaction requested=true at 1731916407670 (+7 ms)Writing region close event to WAL at 1731916407754 (+84 ms)Running coprocessor post-close hooks at 1731916407757 (+3 ms)Closed at 1731916407757 2024-11-18T07:53:27,759 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(157): Closed 1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:27,759 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=1e0c954a4c9c2df78d79f9a8a402493e, regionState=CLOSED 2024-11-18T07:53:27,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:53:27,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-18T07:53:27,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e0c954a4c9c2df78d79f9a8a402493e, server=87fb3d252950,37987,1731916323128 in 2.5780 sec 2024-11-18T07:53:27,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-11-18T07:53:27,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e0c954a4c9c2df78d79f9a8a402493e, UNASSIGN in 2.5860 sec 2024-11-18T07:53:27,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-18T07:53:27,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5890 sec 2024-11-18T07:53:27,767 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916407767"}]},"ts":"1731916407767"} 2024-11-18T07:53:27,769 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T07:53:27,769 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T07:53:27,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.6080 sec 2024-11-18T07:53:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T07:53:29,349 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:29,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T07:53:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:29,352 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T07:53:29,353 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:29,357 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:29,362 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/recovered.edits] 2024-11-18T07:53:29,367 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/4cea19b73b3d4551805660fc5ee427b5 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/4cea19b73b3d4551805660fc5ee427b5 2024-11-18T07:53:29,369 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/5b4ca57974934ef09d45b99c6dbb5f02 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/5b4ca57974934ef09d45b99c6dbb5f02 2024-11-18T07:53:29,371 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/8c5e57dd3ca04267854cce70416d9dce to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/8c5e57dd3ca04267854cce70416d9dce 2024-11-18T07:53:29,373 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/fa83f96e04d548bfa59e6c0b4000c9fd to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/A/fa83f96e04d548bfa59e6c0b4000c9fd 2024-11-18T07:53:29,376 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/153343212a114dc8804b2313d95ece27 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/153343212a114dc8804b2313d95ece27 2024-11-18T07:53:29,377 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/d93c0c8663ed428fbf0056051017d789 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/d93c0c8663ed428fbf0056051017d789 2024-11-18T07:53:29,378 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ed1ec5b002014b8c9c6e975215548244 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ed1ec5b002014b8c9c6e975215548244 2024-11-18T07:53:29,380 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ff86e113806a47c8be0b70a4f189e5fa to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/B/ff86e113806a47c8be0b70a4f189e5fa 2024-11-18T07:53:29,385 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0b802731de15484fa9827dff00d2b142 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/0b802731de15484fa9827dff00d2b142 2024-11-18T07:53:29,386 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/853fd5978f724570917563afb471b59e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/853fd5978f724570917563afb471b59e 2024-11-18T07:53:29,388 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/ce795760156142f8b7f28f8a18a84bd4 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/ce795760156142f8b7f28f8a18a84bd4 2024-11-18T07:53:29,390 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/fca8a02c378545a687c01d50ddccba54 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/C/fca8a02c378545a687c01d50ddccba54 2024-11-18T07:53:29,396 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/recovered.edits/365.seqid to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e/recovered.edits/365.seqid 2024-11-18T07:53:29,397 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/1e0c954a4c9c2df78d79f9a8a402493e 2024-11-18T07:53:29,397 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T07:53:29,403 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:29,407 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T07:53:29,409 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T07:53:29,411 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:29,411 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T07:53:29,411 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731916409411"}]},"ts":"9223372036854775807"} 2024-11-18T07:53:29,414 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T07:53:29,414 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1e0c954a4c9c2df78d79f9a8a402493e, NAME => 'TestAcidGuarantees,,1731916382219.1e0c954a4c9c2df78d79f9a8a402493e.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T07:53:29,414 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T07:53:29,414 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731916409414"}]},"ts":"9223372036854775807"} 2024-11-18T07:53:29,417 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T07:53:29,418 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:29,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-11-18T07:53:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T07:53:29,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T07:53:29,462 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:29,472 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 253), OpenFileDescriptor=453 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1058 (was 733) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1903 (was 2588) 2024-11-18T07:53:29,483 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=1058, ProcessCount=11, AvailableMemoryMB=1903 2024-11-18T07:53:29,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T07:53:29,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T07:53:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:29,487 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T07:53:29,487 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:29,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 79 2024-11-18T07:53:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-18T07:53:29,488 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T07:53:29,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742135_1311 (size=1061) 2024-11-18T07:53:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-18T07:53:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-18T07:53:29,905 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3 2024-11-18T07:53:29,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742136_1312 (size=53) 2024-11-18T07:53:29,963 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:53:29,963 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4a6df03ed24f69072d2b4a08bac0a9a6, disabling compactions & flushes 2024-11-18T07:53:29,963 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:29,963 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:29,963 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. after waiting 0 ms 2024-11-18T07:53:29,963 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:29,963 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:29,963 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4a6df03ed24f69072d2b4a08bac0a9a6: Waiting for close lock at 1731916409963Disabling compacts and flushes for region at 1731916409963Disabling writes for close at 1731916409963Writing region close event to WAL at 1731916409963Closed at 1731916409963 2024-11-18T07:53:29,965 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T07:53:29,965 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731916409965"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731916409965"}]},"ts":"1731916409965"} 2024-11-18T07:53:29,967 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T07:53:29,970 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T07:53:29,970 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916409970"}]},"ts":"1731916409970"} 2024-11-18T07:53:29,973 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T07:53:29,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a6df03ed24f69072d2b4a08bac0a9a6, ASSIGN}] 2024-11-18T07:53:29,976 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a6df03ed24f69072d2b4a08bac0a9a6, ASSIGN 2024-11-18T07:53:29,977 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a6df03ed24f69072d2b4a08bac0a9a6, ASSIGN; state=OFFLINE, location=87fb3d252950,37987,1731916323128; forceNewPlan=false, retain=false 2024-11-18T07:53:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-18T07:53:30,128 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=4a6df03ed24f69072d2b4a08bac0a9a6, regionState=OPENING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:53:30,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a6df03ed24f69072d2b4a08bac0a9a6, ASSIGN because future has completed 2024-11-18T07:53:30,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:53:30,292 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:30,293 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} 2024-11-18T07:53:30,293 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:30,293 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:53:30,293 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:30,293 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:30,295 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:30,297 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:53:30,297 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a6df03ed24f69072d2b4a08bac0a9a6 columnFamilyName A 2024-11-18T07:53:30,297 DEBUG [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:30,298 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.HStore(327): Store=4a6df03ed24f69072d2b4a08bac0a9a6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:53:30,298 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:30,300 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:53:30,300 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a6df03ed24f69072d2b4a08bac0a9a6 columnFamilyName B 2024-11-18T07:53:30,300 DEBUG [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:30,301 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.HStore(327): Store=4a6df03ed24f69072d2b4a08bac0a9a6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:53:30,302 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:30,313 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:53:30,325 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a6df03ed24f69072d2b4a08bac0a9a6 columnFamilyName C 2024-11-18T07:53:30,325 DEBUG [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:30,328 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.HStore(327): Store=4a6df03ed24f69072d2b4a08bac0a9a6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:53:30,329 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:30,329 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:30,334 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:30,335 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:30,336 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:30,336 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:30,337 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T07:53:30,339 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:30,341 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T07:53:30,342 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 4a6df03ed24f69072d2b4a08bac0a9a6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58848362, jitterRate=-0.12309107184410095}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T07:53:30,342 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:30,343 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 4a6df03ed24f69072d2b4a08bac0a9a6: Running coprocessor pre-open hook at 1731916410294Writing region info on filesystem at 1731916410294Initializing all the Stores at 1731916410294Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916410295 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916410295Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916410295Cleaning up temporary data from old regions at 1731916410336 (+41 ms)Running coprocessor post-open hooks at 1731916410342 (+6 ms)Region opened successfully at 1731916410343 (+1 ms) 2024-11-18T07:53:30,344 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., pid=81, masterSystemTime=1731916410283 2024-11-18T07:53:30,348 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:30,348 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:30,348 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=4a6df03ed24f69072d2b4a08bac0a9a6, regionState=OPEN, openSeqNum=2, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:53:30,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:53:30,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-18T07:53:30,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure 4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 in 221 msec 2024-11-18T07:53:30,358 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-18T07:53:30,358 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a6df03ed24f69072d2b4a08bac0a9a6, ASSIGN in 382 msec 2024-11-18T07:53:30,360 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T07:53:30,360 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916410360"}]},"ts":"1731916410360"} 2024-11-18T07:53:30,364 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T07:53:30,365 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T07:53:30,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 880 msec 2024-11-18T07:53:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-18T07:53:30,628 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:30,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11853038, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:30,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:30,629 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:30,631 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c36da5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:30,632 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@538a8ceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:30,633 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:30,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:30,636 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T07:53:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 87fb3d252950,42633,1731916322278 2024-11-18T07:53:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@596e5756 2024-11-18T07:53:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T07:53:30,638 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T07:53:30,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T07:53:30,640 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T07:53:30,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:30,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742137_1313 (size=1097) 2024-11-18T07:53:31,052 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-18T07:53:31,052 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-18T07:53:31,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T07:53:31,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a6df03ed24f69072d2b4a08bac0a9a6, REOPEN/MOVE}] 2024-11-18T07:53:31,058 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a6df03ed24f69072d2b4a08bac0a9a6, REOPEN/MOVE 2024-11-18T07:53:31,058 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=4a6df03ed24f69072d2b4a08bac0a9a6, regionState=CLOSING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:53:31,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a6df03ed24f69072d2b4a08bac0a9a6, REOPEN/MOVE because future has completed 2024-11-18T07:53:31,060 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T07:53:31,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:53:31,212 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,212 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T07:53:31,212 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 4a6df03ed24f69072d2b4a08bac0a9a6, disabling compactions & flushes 2024-11-18T07:53:31,212 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:31,212 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:31,212 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. after waiting 0 ms 2024-11-18T07:53:31,212 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:31,220 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T07:53:31,221 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:31,221 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 4a6df03ed24f69072d2b4a08bac0a9a6: Waiting for close lock at 1731916411212Running coprocessor pre-close hooks at 1731916411212Disabling compacts and flushes for region at 1731916411212Disabling writes for close at 1731916411212Writing region close event to WAL at 1731916411213 (+1 ms)Running coprocessor post-close hooks at 1731916411221 (+8 ms)Closed at 1731916411221 2024-11-18T07:53:31,221 WARN [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: 4a6df03ed24f69072d2b4a08bac0a9a6 to self. 2024-11-18T07:53:31,223 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,224 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=4a6df03ed24f69072d2b4a08bac0a9a6, regionState=CLOSED 2024-11-18T07:53:31,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:53:31,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-18T07:53:31,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 in 165 msec 2024-11-18T07:53:31,228 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a6df03ed24f69072d2b4a08bac0a9a6, REOPEN/MOVE; state=CLOSED, location=87fb3d252950,37987,1731916323128; forceNewPlan=false, retain=true 2024-11-18T07:53:31,236 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T07:53:31,378 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=4a6df03ed24f69072d2b4a08bac0a9a6, regionState=OPENING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:53:31,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a6df03ed24f69072d2b4a08bac0a9a6, REOPEN/MOVE because future has completed 2024-11-18T07:53:31,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:53:31,542 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:31,542 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} 2024-11-18T07:53:31,542 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,543 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:53:31,543 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7794): checking encryption for 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,543 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7797): checking classloading for 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,551 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,552 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:53:31,552 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a6df03ed24f69072d2b4a08bac0a9a6 columnFamilyName A 2024-11-18T07:53:31,554 DEBUG [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:31,555 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.HStore(327): Store=4a6df03ed24f69072d2b4a08bac0a9a6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:53:31,556 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,556 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:53:31,557 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a6df03ed24f69072d2b4a08bac0a9a6 columnFamilyName B 2024-11-18T07:53:31,557 DEBUG [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:31,557 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.HStore(327): Store=4a6df03ed24f69072d2b4a08bac0a9a6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:53:31,557 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,558 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:53:31,558 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a6df03ed24f69072d2b4a08bac0a9a6 columnFamilyName C 2024-11-18T07:53:31,558 DEBUG [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:31,559 INFO [StoreOpener-4a6df03ed24f69072d2b4a08bac0a9a6-1 {}] regionserver.HStore(327): Store=4a6df03ed24f69072d2b4a08bac0a9a6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:53:31,559 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:31,559 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1038): replaying wal for 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,560 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,561 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,562 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1048): stopping wal replay for 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,562 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1060): Cleaning up temporary data for 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,562 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T07:53:31,564 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1093): writing seq id for 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,564 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1114): Opened 4a6df03ed24f69072d2b4a08bac0a9a6; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64017208, jitterRate=-0.04606926441192627}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T07:53:31,564 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,565 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1006): Region open journal for 4a6df03ed24f69072d2b4a08bac0a9a6: Running coprocessor pre-open hook at 1731916411543Writing region info on filesystem at 1731916411543Initializing all the Stores at 1731916411547 (+4 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916411548 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916411550 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916411550Cleaning up temporary data from old regions at 1731916411562 (+12 ms)Running coprocessor post-open hooks at 1731916411564 (+2 ms)Region opened successfully at 1731916411565 (+1 ms) 2024-11-18T07:53:31,566 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., pid=86, masterSystemTime=1731916411532 2024-11-18T07:53:31,569 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:31,569 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:31,569 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=4a6df03ed24f69072d2b4a08bac0a9a6, regionState=OPEN, openSeqNum=5, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:53:31,571 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:53:31,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=84 2024-11-18T07:53:31,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 in 191 msec 2024-11-18T07:53:31,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-18T07:53:31,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a6df03ed24f69072d2b4a08bac0a9a6, REOPEN/MOVE in 516 msec 2024-11-18T07:53:31,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-18T07:53:31,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 521 msec 2024-11-18T07:53:31,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-18T07:53:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T07:53:31,581 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:31,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d5e8400, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,582 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:31,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:31,583 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13504bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:31,585 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:31,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bdb25c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,587 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ce5c6bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,587 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:31,587 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:31,588 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f5d5475, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:31,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:31,590 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41126f45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7fbdbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,593 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:31,593 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:31,594 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:31,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:31,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:31,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6807c5cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:31,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:31,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:31,596 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74371711, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37842ce8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,598 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:31,598 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:31,599 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23bbd73e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:31,601 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c8ae085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2853677b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,603 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:31,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:31,605 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219902cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:31,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:31,606 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:31,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259568f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ad32ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:31,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:31,609 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:31,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:31,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7de9d089, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:31,611 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a6e2d7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@615de503, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:31,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:31,616 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:31,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:31,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d074659, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:31,618 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:31,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71c12fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5690e3aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,623 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:31,623 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:31,624 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:31,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:31,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:31,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9e78c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:31,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:31,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:31,626 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca890dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a170ebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,631 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:31,631 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:31,632 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:31,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a851d3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:31,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:31,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:31,634 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3352f76a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45749f56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:31,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:31,640 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d57e471, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:31,641 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ccbfd50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:31,643 DEBUG [Thread-1454 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:31,643 DEBUG [Thread-1450 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:31,645 DEBUG [Thread-1456 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:31,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:31,645 DEBUG [Thread-1447 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:31,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:31,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:31,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:31,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:53:31,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:53:31,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:53:31,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:53:31,659 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:31,660 DEBUG [Thread-1452 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-18T07:53:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T07:53:31,662 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:31,662 DEBUG [Thread-1452 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:31,662 DEBUG [Thread-1452 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:31,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:31,665 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:31,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:31,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:53:31,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:53:31,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:31,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:31,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:31,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:31,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:31,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,675 DEBUG [Thread-1458 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:31,675 DEBUG [Thread-1439 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:31,676 DEBUG [Thread-1441 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:31,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:31,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:31,678 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:31,678 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:31,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:31,680 DEBUG [Thread-1443 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:31,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:53:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:53:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:53:31,704 DEBUG [Thread-1443 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:31,704 DEBUG [Thread-1443 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:31,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:53:31,712 DEBUG [Thread-1445 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:31,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:31,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:31,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:53:31,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d2a41912e5164383bf92c352cc81798f_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916411664/Put/seqid=0 2024-11-18T07:53:31,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731916411703, max timestamp=1731916411733Num uniques 859; 2024-11-18T07:53:31,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:53:31,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731916411703, max timestamp=1731916411733Num uniques 859; 2024-11-18T07:53:31,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:53:31,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731916411703, max timestamp=1731916411733Num uniques 859; 2024-11-18T07:53:31,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:53:31,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742138_1314 (size=12154) 2024-11-18T07:53:31,749 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:31,753 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d2a41912e5164383bf92c352cc81798f_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d2a41912e5164383bf92c352cc81798f_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:31,754 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/17372032f64d46e69ab3a52d5f6441c0, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:31,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/17372032f64d46e69ab3a52d5f6441c0 is 175, key is test_row_0/A:col10/1731916411664/Put/seqid=0 2024-11-18T07:53:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T07:53:31,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742139_1315 (size=30955) 2024-11-18T07:53:31,769 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/17372032f64d46e69ab3a52d5f6441c0 2024-11-18T07:53:31,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916471797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:31,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:31,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916471798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:31,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:31,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916471799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:31,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T07:53:31,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:31,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:31,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:31,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:31,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:31,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:31,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916471813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:31,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:31,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/ddac30d039f541219759ebcc9ea85a52 is 50, key is test_row_0/B:col10/1731916411664/Put/seqid=0 2024-11-18T07:53:31,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:31,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916471830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:31,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742140_1316 (size=12001) 2024-11-18T07:53:31,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/ddac30d039f541219759ebcc9ea85a52 2024-11-18T07:53:31,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/495bd213ab494c38b73b71f01be6fd9f is 50, key is test_row_0/C:col10/1731916411664/Put/seqid=0 2024-11-18T07:53:31,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742141_1317 (size=12001) 2024-11-18T07:53:31,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:31,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916471926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:31,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:31,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916471927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:31,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:31,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:31,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916471927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:31,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:31,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916471938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:31,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:31,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916471948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:31,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T07:53:31,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:31,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:31,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:31,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:31,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:31,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:31,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T07:53:32,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T07:53:32,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:32,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:32,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:32,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:32,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:32,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:32,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:32,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916472139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:32,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916472139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:32,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916472139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:32,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916472166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:32,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:32,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916472168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:32,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T07:53:32,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:32,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:32,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:32,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:32,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:32,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:32,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T07:53:32,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/495bd213ab494c38b73b71f01be6fd9f 2024-11-18T07:53:32,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/17372032f64d46e69ab3a52d5f6441c0 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/17372032f64d46e69ab3a52d5f6441c0 2024-11-18T07:53:32,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/17372032f64d46e69ab3a52d5f6441c0, entries=150, sequenceid=14, filesize=30.2 K 2024-11-18T07:53:32,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/ddac30d039f541219759ebcc9ea85a52 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/ddac30d039f541219759ebcc9ea85a52 2024-11-18T07:53:32,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/ddac30d039f541219759ebcc9ea85a52, entries=150, sequenceid=14, filesize=11.7 K 2024-11-18T07:53:32,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/495bd213ab494c38b73b71f01be6fd9f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/495bd213ab494c38b73b71f01be6fd9f 2024-11-18T07:53:32,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/495bd213ab494c38b73b71f01be6fd9f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-18T07:53:32,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 678ms, sequenceid=14, compaction requested=false 2024-11-18T07:53:32,350 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T07:53:32,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:32,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T07:53:32,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:32,432 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-18T07:53:32,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:32,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:32,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:32,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:32,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:32,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:32,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f220ad28177649ca90687ac53f7a4dca_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916411813/Put/seqid=0 2024-11-18T07:53:32,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:32,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:32,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742142_1318 (size=12154) 2024-11-18T07:53:32,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:32,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916472473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:32,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916472474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:32,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916472475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:32,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:32,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916472478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:32,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916472478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:32,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:32,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916472599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:32,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:32,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916472599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:32,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916472599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:32,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T07:53:32,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916472808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:32,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916472809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:32,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916472818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:32,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:32,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:32,867 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f220ad28177649ca90687ac53f7a4dca_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f220ad28177649ca90687ac53f7a4dca_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:32,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/95410d9d1a1241a5b30dbbca05789d93, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:32,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/95410d9d1a1241a5b30dbbca05789d93 is 175, key is test_row_0/A:col10/1731916411813/Put/seqid=0 2024-11-18T07:53:32,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742143_1319 (size=30955) 2024-11-18T07:53:33,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916472997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:33,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916472998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:33,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916473118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:33,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:33,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:33,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:33,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:33,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916473128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:33,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:33,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:33,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:33,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916473138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:33,290 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T07:53:33,318 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/95410d9d1a1241a5b30dbbca05789d93 2024-11-18T07:53:33,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/38140b72c77440f38535e1d2989722c3 is 50, key is test_row_0/B:col10/1731916411813/Put/seqid=0 2024-11-18T07:53:33,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742144_1320 (size=12001) 2024-11-18T07:53:33,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916473638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:33,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:33,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:33,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:33,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916473648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:33,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916473658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:33,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:33,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:33,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:33,756 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/38140b72c77440f38535e1d2989722c3 2024-11-18T07:53:33,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/b16dbc85b82545a0a7434b73e032394f is 50, key is test_row_0/C:col10/1731916411813/Put/seqid=0 2024-11-18T07:53:33,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742145_1321 (size=12001) 2024-11-18T07:53:33,802 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/b16dbc85b82545a0a7434b73e032394f 2024-11-18T07:53:33,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/95410d9d1a1241a5b30dbbca05789d93 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/95410d9d1a1241a5b30dbbca05789d93 2024-11-18T07:53:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T07:53:33,824 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/95410d9d1a1241a5b30dbbca05789d93, entries=150, sequenceid=50, filesize=30.2 K 2024-11-18T07:53:33,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/38140b72c77440f38535e1d2989722c3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/38140b72c77440f38535e1d2989722c3 2024-11-18T07:53:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,831 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/38140b72c77440f38535e1d2989722c3, entries=150, sequenceid=50, filesize=11.7 K 2024-11-18T07:53:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/b16dbc85b82545a0a7434b73e032394f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/b16dbc85b82545a0a7434b73e032394f 2024-11-18T07:53:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,839 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/b16dbc85b82545a0a7434b73e032394f, entries=150, sequenceid=50, filesize=11.7 K 2024-11-18T07:53:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,840 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 1408ms, sequenceid=50, compaction requested=false 2024-11-18T07:53:33,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:33,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-18T07:53:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-18T07:53:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-18T07:53:33,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1790 sec 2024-11-18T07:53:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 2.1890 sec 2024-11-18T07:53:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:53:34,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:34,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:34,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:34,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:34,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:34,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:34,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187b9fc140560440148ce261613c2576bc_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916412470/Put/seqid=0 2024-11-18T07:53:34,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742146_1322 (size=12154) 2024-11-18T07:53:34,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916414210, max timestamp=1731916414281Num uniques 859; 2024-11-18T07:53:34,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:53:34,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916414210, max timestamp=1731916414281Num uniques 859; 2024-11-18T07:53:34,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916414210, max timestamp=1731916414281Num uniques 859; 2024-11-18T07:53:34,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:53:34,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:53:34,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916474468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:34,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:34,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916474467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:34,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:34,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:34,537 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:34,544 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187b9fc140560440148ce261613c2576bc_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187b9fc140560440148ce261613c2576bc_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:34,545 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/cb29d4d8f7fd4905820494fa01639b5e, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:34,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/cb29d4d8f7fd4905820494fa01639b5e is 175, key is test_row_0/A:col10/1731916412470/Put/seqid=0 2024-11-18T07:53:34,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742147_1323 (size=30955) 2024-11-18T07:53:34,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916474588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:34,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916474599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:34,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916474658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:34,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916474668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:34,676 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_1' in 4a6df03ed24f69072d2b4a08bac0a9a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:53:34,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:34,677 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_1' in 4a6df03ed24f69072d2b4a08bac0a9a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:53:34,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:34,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916474679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:34,692 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_0' in 4a6df03ed24f69072d2b4a08bac0a9a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:53:34,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:34,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916474808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:34,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916474808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:34,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:34,974 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/cb29d4d8f7fd4905820494fa01639b5e 2024-11-18T07:53:34,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/b671f3095e4348a1a86fb29e6e37caa9 is 50, key is test_row_0/B:col10/1731916412470/Put/seqid=0 2024-11-18T07:53:34,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742148_1324 (size=12001) 2024-11-18T07:53:35,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916475128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:35,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916475128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:35,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/b671f3095e4348a1a86fb29e6e37caa9 2024-11-18T07:53:35,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/5bc955c704d14e08a6d5b86deb15f3c4 is 50, key is test_row_0/C:col10/1731916412470/Put/seqid=0 2024-11-18T07:53:35,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742149_1325 (size=12001) 2024-11-18T07:53:35,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/5bc955c704d14e08a6d5b86deb15f3c4 2024-11-18T07:53:35,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/cb29d4d8f7fd4905820494fa01639b5e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/cb29d4d8f7fd4905820494fa01639b5e 2024-11-18T07:53:35,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/cb29d4d8f7fd4905820494fa01639b5e, entries=150, sequenceid=60, filesize=30.2 K 2024-11-18T07:53:35,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/b671f3095e4348a1a86fb29e6e37caa9 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/b671f3095e4348a1a86fb29e6e37caa9 2024-11-18T07:53:35,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/b671f3095e4348a1a86fb29e6e37caa9, entries=150, sequenceid=60, filesize=11.7 K 2024-11-18T07:53:35,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/5bc955c704d14e08a6d5b86deb15f3c4 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/5bc955c704d14e08a6d5b86deb15f3c4 2024-11-18T07:53:35,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916475640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:35,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:35,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:35,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:35,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916475650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:35,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/5bc955c704d14e08a6d5b86deb15f3c4, entries=150, sequenceid=60, filesize=11.7 K 2024-11-18T07:53:35,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 1606ms, sequenceid=60, compaction requested=true 2024-11-18T07:53:35,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:35,686 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:35,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:53:35,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:35,686 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:35,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:53:35,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:35,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:53:35,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:35,713 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:35,713 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/A is initiating minor compaction (all files) 2024-11-18T07:53:35,714 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/A in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:35,714 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/17372032f64d46e69ab3a52d5f6441c0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/95410d9d1a1241a5b30dbbca05789d93, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/cb29d4d8f7fd4905820494fa01639b5e] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=90.7 K 2024-11-18T07:53:35,714 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:35,714 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/17372032f64d46e69ab3a52d5f6441c0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/95410d9d1a1241a5b30dbbca05789d93, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/cb29d4d8f7fd4905820494fa01639b5e] 2024-11-18T07:53:35,714 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:35,714 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/B is initiating minor compaction (all files) 2024-11-18T07:53:35,714 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/B in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:35,715 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/ddac30d039f541219759ebcc9ea85a52, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/38140b72c77440f38535e1d2989722c3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/b671f3095e4348a1a86fb29e6e37caa9] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=35.2 K 2024-11-18T07:53:35,719 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17372032f64d46e69ab3a52d5f6441c0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731916411664 2024-11-18T07:53:35,719 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting ddac30d039f541219759ebcc9ea85a52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731916411664 2024-11-18T07:53:35,723 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95410d9d1a1241a5b30dbbca05789d93, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731916411774 2024-11-18T07:53:35,723 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 38140b72c77440f38535e1d2989722c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731916411774 2024-11-18T07:53:35,731 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb29d4d8f7fd4905820494fa01639b5e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916412470 2024-11-18T07:53:35,731 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting b671f3095e4348a1a86fb29e6e37caa9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916412470 2024-11-18T07:53:35,805 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#B#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:35,805 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/11ab198a16e14a8595086235cb712716 is 50, key is test_row_0/B:col10/1731916412470/Put/seqid=0 2024-11-18T07:53:35,811 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T07:53:35,831 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118993a27cf6e35477fb02afaae65da4684_4a6df03ed24f69072d2b4a08bac0a9a6 store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:35,833 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118993a27cf6e35477fb02afaae65da4684_4a6df03ed24f69072d2b4a08bac0a9a6, store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:35,833 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118993a27cf6e35477fb02afaae65da4684_4a6df03ed24f69072d2b4a08bac0a9a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:35,835 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:35,843 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-18T07:53:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T07:53:35,857 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:35,859 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:35,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:35,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742150_1326 (size=12104) 2024-11-18T07:53:35,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742151_1327 (size=4469) 2024-11-18T07:53:35,900 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/11ab198a16e14a8595086235cb712716 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/11ab198a16e14a8595086235cb712716 2024-11-18T07:53:35,900 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#A#compaction#277 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:35,901 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/081ca9039ec7489084e8e507326deb37 is 175, key is test_row_0/A:col10/1731916412470/Put/seqid=0 2024-11-18T07:53:35,907 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/B of 4a6df03ed24f69072d2b4a08bac0a9a6 into 11ab198a16e14a8595086235cb712716(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:35,907 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:35,907 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/B, priority=13, startTime=1731916415686; duration=0sec 2024-11-18T07:53:35,907 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:35,907 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:B 2024-11-18T07:53:35,907 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:35,908 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:35,908 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/C is initiating minor compaction (all files) 2024-11-18T07:53:35,908 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/C in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:35,909 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/495bd213ab494c38b73b71f01be6fd9f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/b16dbc85b82545a0a7434b73e032394f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/5bc955c704d14e08a6d5b86deb15f3c4] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=35.2 K 2024-11-18T07:53:35,909 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 495bd213ab494c38b73b71f01be6fd9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731916411664 2024-11-18T07:53:35,909 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting b16dbc85b82545a0a7434b73e032394f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731916411774 2024-11-18T07:53:35,909 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bc955c704d14e08a6d5b86deb15f3c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916412470 2024-11-18T07:53:35,925 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#C#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:35,926 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/8c73c67cedfd4b08938dcd9d6cd5d874 is 50, key is test_row_0/C:col10/1731916412470/Put/seqid=0 2024-11-18T07:53:35,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742152_1328 (size=31058) 2024-11-18T07:53:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T07:53:35,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742153_1329 (size=12104) 2024-11-18T07:53:35,975 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/8c73c67cedfd4b08938dcd9d6cd5d874 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/8c73c67cedfd4b08938dcd9d6cd5d874 2024-11-18T07:53:35,980 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/C of 4a6df03ed24f69072d2b4a08bac0a9a6 into 8c73c67cedfd4b08938dcd9d6cd5d874(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:35,980 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:35,980 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/C, priority=13, startTime=1731916415686; duration=0sec 2024-11-18T07:53:35,980 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:35,980 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:C 2024-11-18T07:53:36,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-18T07:53:36,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:36,022 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-18T07:53:36,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:36,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:36,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:36,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:36,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:36,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:36,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118060cf54a0c9742f4a46471a72bdc3847_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916414465/Put/seqid=0 2024-11-18T07:53:36,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742154_1330 (size=12154) 2024-11-18T07:53:36,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,086 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118060cf54a0c9742f4a46471a72bdc3847_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118060cf54a0c9742f4a46471a72bdc3847_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:36,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/6f2098d967b14870b89a029fc648402f, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:36,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/6f2098d967b14870b89a029fc648402f is 175, key is test_row_0/A:col10/1731916414465/Put/seqid=0 2024-11-18T07:53:36,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742155_1331 (size=30955) 2024-11-18T07:53:36,110 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/6f2098d967b14870b89a029fc648402f 2024-11-18T07:53:36,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/e90cecf607ea4328a023f4ffeb158de2 is 50, key is test_row_0/B:col10/1731916414465/Put/seqid=0 2024-11-18T07:53:36,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742156_1332 (size=12001) 2024-11-18T07:53:36,174 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/e90cecf607ea4328a023f4ffeb158de2 2024-11-18T07:53:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T07:53:36,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/7f59dddd10964800b9c6f4abd02cafa8 is 50, key is test_row_0/C:col10/1731916414465/Put/seqid=0 2024-11-18T07:53:36,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742157_1333 (size=12001) 2024-11-18T07:53:36,300 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/7f59dddd10964800b9c6f4abd02cafa8 2024-11-18T07:53:36,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/6f2098d967b14870b89a029fc648402f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/6f2098d967b14870b89a029fc648402f 2024-11-18T07:53:36,358 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/6f2098d967b14870b89a029fc648402f, entries=150, sequenceid=99, filesize=30.2 K 2024-11-18T07:53:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,367 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/081ca9039ec7489084e8e507326deb37 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/081ca9039ec7489084e8e507326deb37 2024-11-18T07:53:36,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/e90cecf607ea4328a023f4ffeb158de2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/e90cecf607ea4328a023f4ffeb158de2 2024-11-18T07:53:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,377 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/e90cecf607ea4328a023f4ffeb158de2, entries=150, sequenceid=99, filesize=11.7 K 2024-11-18T07:53:36,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/7f59dddd10964800b9c6f4abd02cafa8 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/7f59dddd10964800b9c6f4abd02cafa8 2024-11-18T07:53:36,380 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/A of 4a6df03ed24f69072d2b4a08bac0a9a6 into 081ca9039ec7489084e8e507326deb37(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:36,380 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:36,380 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/A, priority=13, startTime=1731916415686; duration=0sec 2024-11-18T07:53:36,380 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:36,380 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:A 2024-11-18T07:53:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,386 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/7f59dddd10964800b9c6f4abd02cafa8, entries=150, sequenceid=99, filesize=11.7 K 2024-11-18T07:53:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,388 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 365ms, sequenceid=99, compaction requested=false 2024-11-18T07:53:36,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:36,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-18T07:53:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-18T07:53:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-18T07:53:36,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 531 msec 2024-11-18T07:53:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 554 msec 2024-11-18T07:53:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T07:53:36,488 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,491 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-18T07:53:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-18T07:53:36,494 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,498 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:36,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-18T07:53:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-18T07:53:36,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:36,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:36,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:36,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-18T07:53:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-18T07:53:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-18T07:53:36,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 158 msec 2024-11-18T07:53:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 169 msec 2024-11-18T07:53:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-18T07:53:36,812 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-18T07:53:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,821 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-18T07:53:36,832 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:36,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-18T07:53:36,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-18T07:53:36,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:36,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:36,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:36,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-18T07:53:36,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-18T07:53:36,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:36,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-18T07:53:37,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 179 msec 2024-11-18T07:53:37,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 209 msec 2024-11-18T07:53:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-18T07:53:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,152 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:37,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:53:37,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:37,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:37,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:37,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:37,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:37,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:37,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-18T07:53:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T07:53:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,165 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,166 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:37,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111898bd91a046e644cc9cf7e1122c5a12a9_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916417147/Put/seqid=0 2024-11-18T07:53:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742158_1334 (size=21918) 2024-11-18T07:53:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T07:53:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T07:53:37,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:37,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:37,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:37,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:37,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731916417271, max timestamp=1731916417401Num uniques 859; 2024-11-18T07:53:37,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T07:53:37,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731916417271, max timestamp=1731916417401Num uniques 859; 2024-11-18T07:53:37,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731916417271, max timestamp=1731916417401Num uniques 859; 2024-11-18T07:53:37,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T07:53:37,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T07:53:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T07:53:37,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T07:53:37,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:37,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:37,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:37,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:37,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:37,622 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:37,629 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111898bd91a046e644cc9cf7e1122c5a12a9_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111898bd91a046e644cc9cf7e1122c5a12a9_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:37,630 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/c46b882c2eee42c293463636c6701dfc, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:37,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/c46b882c2eee42c293463636c6701dfc is 175, key is test_row_0/A:col10/1731916417147/Put/seqid=0 2024-11-18T07:53:37,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916477550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:37,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916477543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:37,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T07:53:37,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:37,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:37,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:37,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:37,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:37,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:37,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916477602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:37,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:37,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916477624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:37,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:37,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:37,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:37,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742159_1335 (size=65323) 2024-11-18T07:53:37,673 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/c46b882c2eee42c293463636c6701dfc 2024-11-18T07:53:37,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916477633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:37,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/776e4fa444ac49aa88f99e56a038e680 is 50, key is test_row_0/B:col10/1731916417147/Put/seqid=0 2024-11-18T07:53:37,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742160_1336 (size=12001) 2024-11-18T07:53:37,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/776e4fa444ac49aa88f99e56a038e680 2024-11-18T07:53:37,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:37,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916477743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:37,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:37,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916477747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:37,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/c9abc1b2a47c48bdafd54355985403fc is 50, key is test_row_0/C:col10/1731916417147/Put/seqid=0 2024-11-18T07:53:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T07:53:37,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916477758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:37,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916477758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:37,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742161_1337 (size=12001) 2024-11-18T07:53:37,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T07:53:37,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:37,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:37,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:37,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:37,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:37,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/c9abc1b2a47c48bdafd54355985403fc 2024-11-18T07:53:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:37,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:37,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916477799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:37,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/c46b882c2eee42c293463636c6701dfc as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/c46b882c2eee42c293463636c6701dfc 2024-11-18T07:53:37,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/c46b882c2eee42c293463636c6701dfc, entries=350, sequenceid=112, filesize=63.8 K 2024-11-18T07:53:37,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/776e4fa444ac49aa88f99e56a038e680 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/776e4fa444ac49aa88f99e56a038e680 2024-11-18T07:53:37,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/776e4fa444ac49aa88f99e56a038e680, entries=150, sequenceid=112, filesize=11.7 K 2024-11-18T07:53:37,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/c9abc1b2a47c48bdafd54355985403fc as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/c9abc1b2a47c48bdafd54355985403fc 2024-11-18T07:53:37,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/c9abc1b2a47c48bdafd54355985403fc, entries=150, sequenceid=112, filesize=11.7 K 2024-11-18T07:53:37,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 713ms, sequenceid=112, compaction requested=true 2024-11-18T07:53:37,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:53:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:53:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:53:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:53:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T07:53:37,869 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:37,869 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:37,871 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:37,871 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/C is initiating minor compaction (all files) 2024-11-18T07:53:37,871 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/C in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:37,871 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/8c73c67cedfd4b08938dcd9d6cd5d874, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/7f59dddd10964800b9c6f4abd02cafa8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/c9abc1b2a47c48bdafd54355985403fc] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=35.3 K 2024-11-18T07:53:37,871 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127336 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:37,871 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/A is initiating minor compaction (all files) 2024-11-18T07:53:37,871 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/A in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:37,872 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/081ca9039ec7489084e8e507326deb37, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/6f2098d967b14870b89a029fc648402f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/c46b882c2eee42c293463636c6701dfc] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=124.4 K 2024-11-18T07:53:37,872 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:37,872 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/081ca9039ec7489084e8e507326deb37, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/6f2098d967b14870b89a029fc648402f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/c46b882c2eee42c293463636c6701dfc] 2024-11-18T07:53:37,872 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 081ca9039ec7489084e8e507326deb37, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916412470 2024-11-18T07:53:37,872 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c73c67cedfd4b08938dcd9d6cd5d874, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916412470 2024-11-18T07:53:37,873 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f59dddd10964800b9c6f4abd02cafa8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731916414442 2024-11-18T07:53:37,873 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f2098d967b14870b89a029fc648402f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731916414442 2024-11-18T07:53:37,873 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c46b882c2eee42c293463636c6701dfc, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731916417032 2024-11-18T07:53:37,873 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting c9abc1b2a47c48bdafd54355985403fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731916417058 2024-11-18T07:53:37,908 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:37,913 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#C#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:37,914 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/76439e38243c4f2d833c4ee4059fa3c3 is 50, key is test_row_0/C:col10/1731916417147/Put/seqid=0 2024-11-18T07:53:37,915 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118863e509a2f604c9998efd0061a8cf8eb_4a6df03ed24f69072d2b4a08bac0a9a6 store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:37,918 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118863e509a2f604c9998efd0061a8cf8eb_4a6df03ed24f69072d2b4a08bac0a9a6, store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:37,918 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118863e509a2f604c9998efd0061a8cf8eb_4a6df03ed24f69072d2b4a08bac0a9a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:37,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T07:53:37,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:37,974 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-18T07:53:37,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:37,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:37,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:37,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:37,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:37,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:37,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742162_1338 (size=12207) 2024-11-18T07:53:37,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742163_1339 (size=4469) 2024-11-18T07:53:37,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:38,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187f448c2354664b5b9b8687e86c427e4a_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916417504/Put/seqid=0 2024-11-18T07:53:38,007 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#A#compaction#285 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:38,011 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/41a14a2d590f48d496e831d13365a396 is 175, key is test_row_0/A:col10/1731916417147/Put/seqid=0 2024-11-18T07:53:38,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742165_1341 (size=31161) 2024-11-18T07:53:38,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916478031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916478041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916478043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916478044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916478046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742164_1340 (size=12304) 2024-11-18T07:53:38,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,109 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/41a14a2d590f48d496e831d13365a396 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/41a14a2d590f48d496e831d13365a396 2024-11-18T07:53:38,157 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/A of 4a6df03ed24f69072d2b4a08bac0a9a6 into 41a14a2d590f48d496e831d13365a396(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:38,157 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:38,157 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/A, priority=13, startTime=1731916417868; duration=0sec 2024-11-18T07:53:38,157 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:38,157 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:A 2024-11-18T07:53:38,157 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:38,171 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:38,171 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/B is initiating minor compaction (all files) 2024-11-18T07:53:38,171 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/B in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:38,171 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/11ab198a16e14a8595086235cb712716, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/e90cecf607ea4328a023f4ffeb158de2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/776e4fa444ac49aa88f99e56a038e680] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=35.3 K 2024-11-18T07:53:38,175 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11ab198a16e14a8595086235cb712716, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916412470 2024-11-18T07:53:38,179 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e90cecf607ea4328a023f4ffeb158de2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731916414442 2024-11-18T07:53:38,183 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 776e4fa444ac49aa88f99e56a038e680, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731916417058 2024-11-18T07:53:38,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916478208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,241 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#B#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:38,241 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/db004edace994088924838423d8a494e is 50, key is test_row_0/B:col10/1731916417147/Put/seqid=0 2024-11-18T07:53:38,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916478208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916478208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916478209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742166_1342 (size=12207) 2024-11-18T07:53:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T07:53:38,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916478370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,419 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/76439e38243c4f2d833c4ee4059fa3c3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/76439e38243c4f2d833c4ee4059fa3c3 2024-11-18T07:53:38,452 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/C of 4a6df03ed24f69072d2b4a08bac0a9a6 into 76439e38243c4f2d833c4ee4059fa3c3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:38,452 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:38,452 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/C, priority=13, startTime=1731916417869; duration=0sec 2024-11-18T07:53:38,452 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:38,452 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:C 2024-11-18T07:53:38,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916478439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916478449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916478451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916478451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:38,560 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187f448c2354664b5b9b8687e86c427e4a_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187f448c2354664b5b9b8687e86c427e4a_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:38,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/cb07c9f98e404bbc803db2711b53e29a, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:38,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/cb07c9f98e404bbc803db2711b53e29a is 175, key is test_row_0/A:col10/1731916417504/Put/seqid=0 2024-11-18T07:53:38,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742167_1343 (size=31105) 2024-11-18T07:53:38,596 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/cb07c9f98e404bbc803db2711b53e29a 2024-11-18T07:53:38,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/1265f83e0a714cffb9b0130f16189242 is 50, key is test_row_0/B:col10/1731916417504/Put/seqid=0 2024-11-18T07:53:38,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742168_1344 (size=12151) 2024-11-18T07:53:38,782 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/db004edace994088924838423d8a494e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/db004edace994088924838423d8a494e 2024-11-18T07:53:38,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916478768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916478789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916478789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916478800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:38,859 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/B of 4a6df03ed24f69072d2b4a08bac0a9a6 into db004edace994088924838423d8a494e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:38,859 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:38,859 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/B, priority=13, startTime=1731916417869; duration=0sec 2024-11-18T07:53:38,859 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:38,859 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:B 2024-11-18T07:53:38,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:38,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916478911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:39,090 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/1265f83e0a714cffb9b0130f16189242 2024-11-18T07:53:39,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/678e5430fceb433693761ee35e213db5 is 50, key is test_row_0/C:col10/1731916417504/Put/seqid=0 2024-11-18T07:53:39,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742169_1345 (size=12151) 2024-11-18T07:53:39,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916479308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T07:53:39,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916479318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:39,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916479319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:39,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916479338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:39,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:39,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:39,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:39,598 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/678e5430fceb433693761ee35e213db5 2024-11-18T07:53:39,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/cb07c9f98e404bbc803db2711b53e29a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/cb07c9f98e404bbc803db2711b53e29a 2024-11-18T07:53:39,615 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/cb07c9f98e404bbc803db2711b53e29a, entries=150, sequenceid=147, filesize=30.4 K 2024-11-18T07:53:39,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/1265f83e0a714cffb9b0130f16189242 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1265f83e0a714cffb9b0130f16189242 2024-11-18T07:53:39,623 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1265f83e0a714cffb9b0130f16189242, entries=150, sequenceid=147, filesize=11.9 K 2024-11-18T07:53:39,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/678e5430fceb433693761ee35e213db5 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/678e5430fceb433693761ee35e213db5 2024-11-18T07:53:39,636 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/678e5430fceb433693761ee35e213db5, entries=150, sequenceid=147, filesize=11.9 K 2024-11-18T07:53:39,638 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 1664ms, sequenceid=147, compaction requested=false 2024-11-18T07:53:39,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:39,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:39,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-18T07:53:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-18T07:53:39,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-18T07:53:39,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4840 sec 2024-11-18T07:53:39,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.5070 sec 2024-11-18T07:53:39,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T07:53:39,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:39,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:39,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:39,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:39,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:39,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:39,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181344127826ca4920ba76ed4e480bd13f_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916418043/Put/seqid=0 2024-11-18T07:53:40,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742170_1346 (size=14794) 2024-11-18T07:53:40,039 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:40,061 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181344127826ca4920ba76ed4e480bd13f_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181344127826ca4920ba76ed4e480bd13f_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:40,066 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/91af8157a6bb4c24ab0f783710ba6644, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:40,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/91af8157a6bb4c24ab0f783710ba6644 is 175, key is test_row_0/A:col10/1731916418043/Put/seqid=0 2024-11-18T07:53:40,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742171_1347 (size=39749) 2024-11-18T07:53:40,110 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/91af8157a6bb4c24ab0f783710ba6644 2024-11-18T07:53:40,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/b94685e1ee6845e2bb4481b8325129ba is 50, key is test_row_0/B:col10/1731916418043/Put/seqid=0 2024-11-18T07:53:40,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742172_1348 (size=12151) 2024-11-18T07:53:40,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916480374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:40,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916480384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:40,423 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_2' in 4a6df03ed24f69072d2b4a08bac0a9a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2380 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:53:40,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:40,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731916420286, max timestamp=1731916420415Num uniques 859; 2024-11-18T07:53:40,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T07:53:40,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731916420286, max timestamp=1731916420415Num uniques 859; 2024-11-18T07:53:40,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T07:53:40,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731916420286, max timestamp=1731916420415Num uniques 859; 2024-11-18T07:53:40,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T07:53:40,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/b94685e1ee6845e2bb4481b8325129ba 2024-11-18T07:53:40,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:40,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916480623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916480612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:40,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916480623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:40,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916480625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:40,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/f5c4fa0a637c4a138f4abb90a8e8a53d is 50, key is test_row_0/C:col10/1731916418043/Put/seqid=0 2024-11-18T07:53:40,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742173_1349 (size=12151) 2024-11-18T07:53:40,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916480768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:40,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916480768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:40,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916480768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:40,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916480768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:41,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916481009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916481009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916481009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916481009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/f5c4fa0a637c4a138f4abb90a8e8a53d 2024-11-18T07:53:41,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/91af8157a6bb4c24ab0f783710ba6644 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/91af8157a6bb4c24ab0f783710ba6644 2024-11-18T07:53:41,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/91af8157a6bb4c24ab0f783710ba6644, entries=200, sequenceid=161, filesize=38.8 K 2024-11-18T07:53:41,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/b94685e1ee6845e2bb4481b8325129ba as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/b94685e1ee6845e2bb4481b8325129ba 2024-11-18T07:53:41,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/b94685e1ee6845e2bb4481b8325129ba, entries=150, sequenceid=161, filesize=11.9 K 2024-11-18T07:53:41,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/f5c4fa0a637c4a138f4abb90a8e8a53d as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/f5c4fa0a637c4a138f4abb90a8e8a53d 2024-11-18T07:53:41,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/f5c4fa0a637c4a138f4abb90a8e8a53d, entries=150, sequenceid=161, filesize=11.9 K 2024-11-18T07:53:41,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=126.28 KB/129315 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 1215ms, sequenceid=161, compaction requested=true 2024-11-18T07:53:41,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:41,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:53:41,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:41,177 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:41,177 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:41,179 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:41,179 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:41,179 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/A is initiating minor compaction (all files) 2024-11-18T07:53:41,179 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/B is initiating minor compaction (all files) 2024-11-18T07:53:41,179 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/B in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:41,179 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/db004edace994088924838423d8a494e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1265f83e0a714cffb9b0130f16189242, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/b94685e1ee6845e2bb4481b8325129ba] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=35.7 K 2024-11-18T07:53:41,179 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/A in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:41,179 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/41a14a2d590f48d496e831d13365a396, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/cb07c9f98e404bbc803db2711b53e29a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/91af8157a6bb4c24ab0f783710ba6644] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=99.6 K 2024-11-18T07:53:41,179 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:41,179 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/41a14a2d590f48d496e831d13365a396, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/cb07c9f98e404bbc803db2711b53e29a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/91af8157a6bb4c24ab0f783710ba6644] 2024-11-18T07:53:41,180 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41a14a2d590f48d496e831d13365a396, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731916417058 2024-11-18T07:53:41,180 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting db004edace994088924838423d8a494e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731916417058 2024-11-18T07:53:41,180 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb07c9f98e404bbc803db2711b53e29a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731916417504 2024-11-18T07:53:41,180 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1265f83e0a714cffb9b0130f16189242, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731916417504 2024-11-18T07:53:41,181 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91af8157a6bb4c24ab0f783710ba6644, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731916417992 2024-11-18T07:53:41,181 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting b94685e1ee6845e2bb4481b8325129ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731916417992 2024-11-18T07:53:41,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:53:41,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:41,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:53:41,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:41,201 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:41,201 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/1683bd52dcab44b79e159175ff007b6a is 50, key is test_row_0/B:col10/1731916418043/Put/seqid=0 2024-11-18T07:53:41,219 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:41,259 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111875e24f8a0e91432f9dd961dfbdf1742a_4a6df03ed24f69072d2b4a08bac0a9a6 store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:41,261 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111875e24f8a0e91432f9dd961dfbdf1742a_4a6df03ed24f69072d2b4a08bac0a9a6, store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:41,262 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111875e24f8a0e91432f9dd961dfbdf1742a_4a6df03ed24f69072d2b4a08bac0a9a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:41,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742174_1350 (size=12459) 2024-11-18T07:53:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T07:53:41,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742175_1351 (size=4469) 2024-11-18T07:53:41,356 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:41,357 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#A#compaction#295 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:41,358 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/5f6152fbe99b49819deb6ee00d3c9154 is 175, key is test_row_0/A:col10/1731916418043/Put/seqid=0 2024-11-18T07:53:41,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-18T07:53:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T07:53:41,369 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:41,370 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:41,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:41,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=132.99 KB heapSize=382.24 KB 2024-11-18T07:53:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:41,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:41,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T07:53:41,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:41,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T07:53:41,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:41,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T07:53:41,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742176_1352 (size=31413) 2024-11-18T07:53:41,417 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/5f6152fbe99b49819deb6ee00d3c9154 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/5f6152fbe99b49819deb6ee00d3c9154 2024-11-18T07:53:41,422 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/A of 4a6df03ed24f69072d2b4a08bac0a9a6 into 5f6152fbe99b49819deb6ee00d3c9154(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:41,422 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:41,422 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/A, priority=13, startTime=1731916421177; duration=0sec 2024-11-18T07:53:41,423 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:41,423 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:A 2024-11-18T07:53:41,423 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:41,440 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:41,440 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/C is initiating minor compaction (all files) 2024-11-18T07:53:41,440 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/C in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:41,440 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/76439e38243c4f2d833c4ee4059fa3c3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/678e5430fceb433693761ee35e213db5, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/f5c4fa0a637c4a138f4abb90a8e8a53d] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=35.7 K 2024-11-18T07:53:41,441 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76439e38243c4f2d833c4ee4059fa3c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731916417058 2024-11-18T07:53:41,441 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 678e5430fceb433693761ee35e213db5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731916417504 2024-11-18T07:53:41,441 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c4fa0a637c4a138f4abb90a8e8a53d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731916417992 2024-11-18T07:53:41,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ac54a475216348aa998f0cb79113bd7c_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916421355/Put/seqid=0 2024-11-18T07:53:41,467 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#C#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:41,468 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/537af44619704fd39c7729f0e8cebfb8 is 50, key is test_row_0/C:col10/1731916418043/Put/seqid=0 2024-11-18T07:53:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T07:53:41,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916481464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916481482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:41,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742177_1353 (size=14794) 2024-11-18T07:53:41,516 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:41,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916481483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:41,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916481485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:41,520 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ac54a475216348aa998f0cb79113bd7c_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ac54a475216348aa998f0cb79113bd7c_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:41,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,521 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/479da5a9af694584beddd79245c5fe07, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:41,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/479da5a9af694584beddd79245c5fe07 is 175, key is test_row_0/A:col10/1731916421355/Put/seqid=0 2024-11-18T07:53:41,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T07:53:41,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:41,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:41,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:41,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:41,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:41,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742178_1354 (size=12459) 2024-11-18T07:53:41,536 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/537af44619704fd39c7729f0e8cebfb8 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/537af44619704fd39c7729f0e8cebfb8 2024-11-18T07:53:41,543 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/C of 4a6df03ed24f69072d2b4a08bac0a9a6 into 537af44619704fd39c7729f0e8cebfb8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:41,543 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:41,543 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/C, priority=13, startTime=1731916421189; duration=0sec 2024-11-18T07:53:41,543 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:41,543 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:C 2024-11-18T07:53:41,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742179_1355 (size=39749) 2024-11-18T07:53:41,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:41,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916481598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:41,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:41,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:41,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916481629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:41,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:41,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916481629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:41,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916481618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T07:53:41,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:41,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:41,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:41,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:41,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:41,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T07:53:41,743 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/1683bd52dcab44b79e159175ff007b6a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1683bd52dcab44b79e159175ff007b6a 2024-11-18T07:53:41,779 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/B of 4a6df03ed24f69072d2b4a08bac0a9a6 into 1683bd52dcab44b79e159175ff007b6a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:41,779 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:41,779 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/B, priority=13, startTime=1731916421177; duration=0sec 2024-11-18T07:53:41,779 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:41,779 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:B 2024-11-18T07:53:41,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T07:53:41,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:41,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:41,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:41,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:41,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:41,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916481818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916481858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916481859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:41,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916481868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:41,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:41,975 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=46.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/479da5a9af694584beddd79245c5fe07 2024-11-18T07:53:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T07:53:42,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T07:53:42,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:42,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:42,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:42,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/1e488c11ddde43ec8a16c886160f8076 is 50, key is test_row_0/B:col10/1731916421355/Put/seqid=0 2024-11-18T07:53:42,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742180_1356 (size=12151) 2024-11-18T07:53:42,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T07:53:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:42,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916482161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:42,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916482201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:42,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916482211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:42,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916482218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:42,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T07:53:42,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:42,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:42,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:42,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916482450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:42,467 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_2' in 4a6df03ed24f69072d2b4a08bac0a9a6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4424 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:53:42,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:42,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T07:53:42,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:42,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:42,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:42,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/1e488c11ddde43ec8a16c886160f8076 2024-11-18T07:53:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T07:53:42,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/4b9134b8cab64419ba910a92e1c8e853 is 50, key is test_row_0/C:col10/1731916421355/Put/seqid=0 2024-11-18T07:53:42,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742181_1357 (size=12151) 2024-11-18T07:53:42,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T07:53:42,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:42,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:42,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:42,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916482688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:42,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916482741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:42,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916482742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:42,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916482748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:42,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:42,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:42,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T07:53:42,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:42,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:42,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:42,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T07:53:42,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:42,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:42,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:42,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:42,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/4b9134b8cab64419ba910a92e1c8e853 2024-11-18T07:53:42,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/479da5a9af694584beddd79245c5fe07 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/479da5a9af694584beddd79245c5fe07 2024-11-18T07:53:42,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/479da5a9af694584beddd79245c5fe07, entries=200, sequenceid=199, filesize=38.8 K 2024-11-18T07:53:42,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/1e488c11ddde43ec8a16c886160f8076 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1e488c11ddde43ec8a16c886160f8076 2024-11-18T07:53:42,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1e488c11ddde43ec8a16c886160f8076, entries=150, sequenceid=199, filesize=11.9 K 2024-11-18T07:53:42,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/4b9134b8cab64419ba910a92e1c8e853 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/4b9134b8cab64419ba910a92e1c8e853 2024-11-18T07:53:43,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/4b9134b8cab64419ba910a92e1c8e853, entries=150, sequenceid=199, filesize=11.9 K 2024-11-18T07:53:43,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~139.70 KB/143055, heapSize ~402.82 KB/412488, currentSize=40.25 KB/41220 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 1626ms, sequenceid=199, compaction requested=false 2024-11-18T07:53:43,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:43,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T07:53:43,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:43,125 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-18T07:53:43,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:43,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:43,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:43,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:43,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:43,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:43,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c0426c31a9af4f8099dd64df22553f17_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916421478/Put/seqid=0 2024-11-18T07:53:43,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742182_1358 (size=12304) 2024-11-18T07:53:43,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:43,224 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c0426c31a9af4f8099dd64df22553f17_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c0426c31a9af4f8099dd64df22553f17_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:43,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/97b904922f944fc0b572fafa1037b4c7, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:43,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/97b904922f944fc0b572fafa1037b4c7 is 175, key is test_row_0/A:col10/1731916421478/Put/seqid=0 2024-11-18T07:53:43,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742183_1359 (size=31105) 2024-11-18T07:53:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T07:53:43,669 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/97b904922f944fc0b572fafa1037b4c7 2024-11-18T07:53:43,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/25b9f7870e86410e942bee06f97e85bc is 50, key is test_row_0/B:col10/1731916421478/Put/seqid=0 2024-11-18T07:53:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:43,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:43,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742184_1360 (size=12151) 2024-11-18T07:53:43,744 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/25b9f7870e86410e942bee06f97e85bc 2024-11-18T07:53:43,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/53be7f525eac4e09b9fb096c62748851 is 50, key is test_row_0/C:col10/1731916421478/Put/seqid=0 2024-11-18T07:53:43,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742185_1361 (size=12151) 2024-11-18T07:53:43,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916483902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:43,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916483910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:43,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916483910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:43,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:43,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:43,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:43,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916483929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:44,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916484038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:44,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916484049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:44,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916484049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:44,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:44,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916484058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:44,239 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/53be7f525eac4e09b9fb096c62748851 2024-11-18T07:53:44,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916484259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:44,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/97b904922f944fc0b572fafa1037b4c7 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/97b904922f944fc0b572fafa1037b4c7 2024-11-18T07:53:44,319 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/97b904922f944fc0b572fafa1037b4c7, entries=150, sequenceid=211, filesize=30.4 K 2024-11-18T07:53:44,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916484291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916484291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:44,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916484291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:44,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/25b9f7870e86410e942bee06f97e85bc as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/25b9f7870e86410e942bee06f97e85bc 2024-11-18T07:53:44,361 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/25b9f7870e86410e942bee06f97e85bc, entries=150, sequenceid=211, filesize=11.9 K 2024-11-18T07:53:44,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/53be7f525eac4e09b9fb096c62748851 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/53be7f525eac4e09b9fb096c62748851 2024-11-18T07:53:44,372 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/53be7f525eac4e09b9fb096c62748851, entries=150, sequenceid=211, filesize=11.9 K 2024-11-18T07:53:44,373 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=134.18 KB/137400 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 1248ms, sequenceid=211, compaction requested=true 2024-11-18T07:53:44,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:44,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:44,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-18T07:53:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-18T07:53:44,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-18T07:53:44,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0050 sec 2024-11-18T07:53:44,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 3.0190 sec 2024-11-18T07:53:44,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=140.89 KB heapSize=419.81 KB 2024-11-18T07:53:44,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:44,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:44,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:44,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:44,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:44,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:44,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185fd0b00b627f489c8f3efc84332b8c85_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916424609/Put/seqid=0 2024-11-18T07:53:44,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916484644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:44,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:44,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916484661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916484663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:44,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916484667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:44,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742186_1362 (size=14794) 2024-11-18T07:53:44,705 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:44,713 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185fd0b00b627f489c8f3efc84332b8c85_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185fd0b00b627f489c8f3efc84332b8c85_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:44,714 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/b0dc4fb934ae475687e83104d134f742, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:44,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/b0dc4fb934ae475687e83104d134f742 is 175, key is test_row_0/A:col10/1731916424609/Put/seqid=0 2024-11-18T07:53:44,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742187_1363 (size=39749) 2024-11-18T07:53:44,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916484779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:44,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916484798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:44,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:44,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916484798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:44,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:44,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:44,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916484798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:45,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:45,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916484988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:45,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:45,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:45,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916485018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:45,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:45,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:45,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916485018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:45,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:45,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:45,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916485019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:45,165 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/b0dc4fb934ae475687e83104d134f742 2024-11-18T07:53:45,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/81f96b6220d94268b53cbadea38785f3 is 50, key is test_row_0/B:col10/1731916424609/Put/seqid=0 2024-11-18T07:53:45,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742188_1364 (size=12151) 2024-11-18T07:53:45,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/81f96b6220d94268b53cbadea38785f3 2024-11-18T07:53:45,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/fc8a6921de8b4783a6751994cb0aeef8 is 50, key is test_row_0/C:col10/1731916424609/Put/seqid=0 2024-11-18T07:53:45,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742189_1365 (size=12151) 2024-11-18T07:53:45,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/fc8a6921de8b4783a6751994cb0aeef8 2024-11-18T07:53:45,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/b0dc4fb934ae475687e83104d134f742 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/b0dc4fb934ae475687e83104d134f742 2024-11-18T07:53:45,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916485315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:45,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:45,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/b0dc4fb934ae475687e83104d134f742, entries=200, sequenceid=235, filesize=38.8 K 2024-11-18T07:53:45,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/81f96b6220d94268b53cbadea38785f3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/81f96b6220d94268b53cbadea38785f3 2024-11-18T07:53:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/81f96b6220d94268b53cbadea38785f3, entries=150, sequenceid=235, filesize=11.9 K 2024-11-18T07:53:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/fc8a6921de8b4783a6751994cb0aeef8 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/fc8a6921de8b4783a6751994cb0aeef8 2024-11-18T07:53:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/fc8a6921de8b4783a6751994cb0aeef8, entries=150, sequenceid=235, filesize=11.9 K 2024-11-18T07:53:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~419.77 KB/429840, currentSize=33.54 KB/34350 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 745ms, sequenceid=235, compaction requested=true 2024-11-18T07:53:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:45,361 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:53:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:53:45,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:45,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:53:45,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:45,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:53:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,362 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:53:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,364 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:53:45,364 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/B is initiating minor compaction (all files) 2024-11-18T07:53:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,364 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/B in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:45,364 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1683bd52dcab44b79e159175ff007b6a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1e488c11ddde43ec8a16c886160f8076, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/25b9f7870e86410e942bee06f97e85bc, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/81f96b6220d94268b53cbadea38785f3] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=47.8 K 2024-11-18T07:53:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,364 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:53:45,364 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/A is initiating minor compaction (all files) 2024-11-18T07:53:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,365 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/A in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:45,365 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/5f6152fbe99b49819deb6ee00d3c9154, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/479da5a9af694584beddd79245c5fe07, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/97b904922f944fc0b572fafa1037b4c7, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/b0dc4fb934ae475687e83104d134f742] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=138.7 K 2024-11-18T07:53:45,365 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:45,365 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/5f6152fbe99b49819deb6ee00d3c9154, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/479da5a9af694584beddd79245c5fe07, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/97b904922f944fc0b572fafa1037b4c7, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/b0dc4fb934ae475687e83104d134f742] 2024-11-18T07:53:45,365 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1683bd52dcab44b79e159175ff007b6a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731916417992 2024-11-18T07:53:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,365 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f6152fbe99b49819deb6ee00d3c9154, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731916417992 2024-11-18T07:53:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,365 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e488c11ddde43ec8a16c886160f8076, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731916420621 2024-11-18T07:53:45,366 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 479da5a9af694584beddd79245c5fe07, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731916420565 2024-11-18T07:53:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,366 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 25b9f7870e86410e942bee06f97e85bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731916421462 2024-11-18T07:53:45,366 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97b904922f944fc0b572fafa1037b4c7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731916421462 2024-11-18T07:53:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,366 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 81f96b6220d94268b53cbadea38785f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731916423823 2024-11-18T07:53:45,366 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0dc4fb934ae475687e83104d134f742, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731916423823 2024-11-18T07:53:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,383 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,403 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#B#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:45,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:45,403 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/5c7d2764a4534a53acac7cb81bf8758e is 50, key is test_row_0/B:col10/1731916424609/Put/seqid=0 2024-11-18T07:53:45,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:53:45,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:45,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:45,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:45,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:45,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:45,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,406 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411187d4ca7fc327a4d5e92da89687c282f59_4a6df03ed24f69072d2b4a08bac0a9a6 store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,409 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411187d4ca7fc327a4d5e92da89687c282f59_4a6df03ed24f69072d2b4a08bac0a9a6, store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,409 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187d4ca7fc327a4d5e92da89687c282f59_4a6df03ed24f69072d2b4a08bac0a9a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742190_1366 (size=12595) 2024-11-18T07:53:45,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742191_1367 (size=4469) 2024-11-18T07:53:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,420 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#A#compaction#306 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:45,420 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/a706b0a02c3942b8a312a809ebef9bf5 is 175, key is test_row_0/A:col10/1731916424609/Put/seqid=0 2024-11-18T07:53:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184999581e58304487ba241b9562e4fc37_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916425402/Put/seqid=0 2024-11-18T07:53:45,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,428 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/5c7d2764a4534a53acac7cb81bf8758e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/5c7d2764a4534a53acac7cb81bf8758e 2024-11-18T07:53:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,442 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/B of 4a6df03ed24f69072d2b4a08bac0a9a6 into 5c7d2764a4534a53acac7cb81bf8758e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:45,442 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:45,442 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/B, priority=12, startTime=1731916425362; duration=0sec 2024-11-18T07:53:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,442 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:45,442 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:B 2024-11-18T07:53:45,442 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:53:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742193_1369 (size=14794) 2024-11-18T07:53:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,459 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:53:45,459 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/C is initiating minor compaction (all files) 2024-11-18T07:53:45,459 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/C in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:45,459 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/537af44619704fd39c7729f0e8cebfb8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/4b9134b8cab64419ba910a92e1c8e853, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/53be7f525eac4e09b9fb096c62748851, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/fc8a6921de8b4783a6751994cb0aeef8] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=47.8 K 2024-11-18T07:53:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,463 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 537af44619704fd39c7729f0e8cebfb8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731916417992 2024-11-18T07:53:45,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742192_1368 (size=31549) 2024-11-18T07:53:45,463 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b9134b8cab64419ba910a92e1c8e853, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731916420621 2024-11-18T07:53:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,467 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 53be7f525eac4e09b9fb096c62748851, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731916421462 2024-11-18T07:53:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,471 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting fc8a6921de8b4783a6751994cb0aeef8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731916423823 2024-11-18T07:53:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,496 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#C#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,496 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/7c60a93d8e1f4edaafb48569526910c0 is 50, key is test_row_0/C:col10/1731916424609/Put/seqid=0 2024-11-18T07:53:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T07:53:45,552 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:45,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742194_1370 (size=12595) 2024-11-18T07:53:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-18T07:53:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,559 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:45,560 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:45,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T07:53:45,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,571 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/7c60a93d8e1f4edaafb48569526910c0 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/7c60a93d8e1f4edaafb48569526910c0 2024-11-18T07:53:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,579 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/C of 4a6df03ed24f69072d2b4a08bac0a9a6 into 7c60a93d8e1f4edaafb48569526910c0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:45,579 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,579 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/C, priority=12, startTime=1731916425362; duration=0sec 2024-11-18T07:53:45,579 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:45,579 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:C 2024-11-18T07:53:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916425630, max timestamp=1731916425652Num uniques 859; 2024-11-18T07:53:45,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916425630, max timestamp=1731916425652Num uniques 859; 2024-11-18T07:53:45,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:53:45,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:53:45,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916425630, max timestamp=1731916425652Num uniques 859; 2024-11-18T07:53:45,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:53:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T07:53:45,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T07:53:45,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:45,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:45,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:45,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:45,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:45,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916485710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:45,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916485714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:45,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916485723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:45,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916485830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:45,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916485838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:45,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:45,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916485838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:45,855 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:45,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:45,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916485853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:45,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T07:53:45,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:45,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:45,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:45,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:45,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T07:53:45,881 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184999581e58304487ba241b9562e4fc37_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184999581e58304487ba241b9562e4fc37_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:45,883 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/a706b0a02c3942b8a312a809ebef9bf5 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/a706b0a02c3942b8a312a809ebef9bf5 2024-11-18T07:53:45,884 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/63a445af98c64746b9f33b371811f8d6, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:45,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/63a445af98c64746b9f33b371811f8d6 is 175, key is test_row_0/A:col10/1731916425402/Put/seqid=0 2024-11-18T07:53:45,891 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/A of 4a6df03ed24f69072d2b4a08bac0a9a6 into a706b0a02c3942b8a312a809ebef9bf5(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:45,891 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:45,891 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/A, priority=12, startTime=1731916425361; duration=0sec 2024-11-18T07:53:45,891 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:45,891 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:A 2024-11-18T07:53:45,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742195_1371 (size=39745) 2024-11-18T07:53:45,922 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/63a445af98c64746b9f33b371811f8d6 2024-11-18T07:53:45,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/9d257b625aee4e92910b00f1b0fcf05d is 50, key is test_row_0/B:col10/1731916425402/Put/seqid=0 2024-11-18T07:53:45,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742196_1372 (size=9757) 2024-11-18T07:53:46,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T07:53:46,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:46,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:46,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:46,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:46,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:46,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916486048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:46,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:46,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916486059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916486068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T07:53:46,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T07:53:46,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:46,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:46,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:46,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:46,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:46,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T07:53:46,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:46,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:46,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:46,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:46,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:46,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/9d257b625aee4e92910b00f1b0fcf05d 2024-11-18T07:53:46,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916486379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:46,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:46,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916486388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:46,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916486399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:46,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/ff3700dcd42349349cb5ac9de3c037b8 is 50, key is test_row_0/C:col10/1731916425402/Put/seqid=0 2024-11-18T07:53:46,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742197_1373 (size=9757) 2024-11-18T07:53:46,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/ff3700dcd42349349cb5ac9de3c037b8 2024-11-18T07:53:46,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/63a445af98c64746b9f33b371811f8d6 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/63a445af98c64746b9f33b371811f8d6 2024-11-18T07:53:46,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/63a445af98c64746b9f33b371811f8d6, entries=200, sequenceid=245, filesize=38.8 K 2024-11-18T07:53:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/9d257b625aee4e92910b00f1b0fcf05d as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/9d257b625aee4e92910b00f1b0fcf05d 2024-11-18T07:53:46,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/9d257b625aee4e92910b00f1b0fcf05d, entries=100, sequenceid=245, filesize=9.5 K 2024-11-18T07:53:46,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/ff3700dcd42349349cb5ac9de3c037b8 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/ff3700dcd42349349cb5ac9de3c037b8 2024-11-18T07:53:46,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/ff3700dcd42349349cb5ac9de3c037b8, entries=100, sequenceid=245, filesize=9.5 K 2024-11-18T07:53:46,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 1086ms, sequenceid=245, compaction requested=false 2024-11-18T07:53:46,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:46,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T07:53:46,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:46,511 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-18T07:53:46,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:46,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:46,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:46,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:46,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:46,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:46,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:46,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184deeb3e5b1974dac9593eea965528734_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916425708/Put/seqid=0 2024-11-18T07:53:46,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742198_1374 (size=12454) 2024-11-18T07:53:46,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:46,598 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184deeb3e5b1974dac9593eea965528734_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184deeb3e5b1974dac9593eea965528734_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:46,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/9c7b6e3d98d44261bdb31d6e487b662b, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:46,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/9c7b6e3d98d44261bdb31d6e487b662b is 175, key is test_row_0/A:col10/1731916425708/Put/seqid=0 2024-11-18T07:53:46,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742199_1375 (size=31255) 2024-11-18T07:53:46,619 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/9c7b6e3d98d44261bdb31d6e487b662b 2024-11-18T07:53:46,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916486643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:46,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/873f9184bf5b45018c8a8a225eecfad4 is 50, key is test_row_0/B:col10/1731916425708/Put/seqid=0 2024-11-18T07:53:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T07:53:46,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742200_1376 (size=12301) 2024-11-18T07:53:46,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916486778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:46,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:46,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916486868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:46,880 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_2' in 4a6df03ed24f69072d2b4a08bac0a9a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:53:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:46,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916486898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:46,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916486908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:46,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916486923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:47,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916487008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:47,140 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/873f9184bf5b45018c8a8a225eecfad4 2024-11-18T07:53:47,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/81e926bed7084fd8a32828749c3ae683 is 50, key is test_row_0/C:col10/1731916425708/Put/seqid=0 2024-11-18T07:53:47,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742201_1377 (size=12301) 2024-11-18T07:53:47,194 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/81e926bed7084fd8a32828749c3ae683 2024-11-18T07:53:47,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/9c7b6e3d98d44261bdb31d6e487b662b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/9c7b6e3d98d44261bdb31d6e487b662b 2024-11-18T07:53:47,202 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/9c7b6e3d98d44261bdb31d6e487b662b, entries=150, sequenceid=285, filesize=30.5 K 2024-11-18T07:53:47,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/873f9184bf5b45018c8a8a225eecfad4 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/873f9184bf5b45018c8a8a225eecfad4 2024-11-18T07:53:47,214 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/873f9184bf5b45018c8a8a225eecfad4, entries=150, sequenceid=285, filesize=12.0 K 2024-11-18T07:53:47,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/81e926bed7084fd8a32828749c3ae683 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/81e926bed7084fd8a32828749c3ae683 2024-11-18T07:53:47,235 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/81e926bed7084fd8a32828749c3ae683, entries=150, sequenceid=285, filesize=12.0 K 2024-11-18T07:53:47,236 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 725ms, sequenceid=285, compaction requested=true 2024-11-18T07:53:47,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:47,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:47,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-18T07:53:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-18T07:53:47,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.6870 sec 2024-11-18T07:53:47,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-18T07:53:47,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6790 sec 2024-11-18T07:53:47,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T07:53:47,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:47,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:47,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:47,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:47,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:47,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:47,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:47,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e1084db790ad4de3b33a3ebb7012881e_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916427335/Put/seqid=0 2024-11-18T07:53:47,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742202_1378 (size=14994) 2024-11-18T07:53:47,394 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:47,407 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e1084db790ad4de3b33a3ebb7012881e_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e1084db790ad4de3b33a3ebb7012881e_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:47,415 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/0c6e6efb8f164168bd06710a9181db37, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:47,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/0c6e6efb8f164168bd06710a9181db37 is 175, key is test_row_0/A:col10/1731916427335/Put/seqid=0 2024-11-18T07:53:47,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742203_1379 (size=39949) 2024-11-18T07:53:47,449 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/0c6e6efb8f164168bd06710a9181db37 2024-11-18T07:53:47,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/3873b390e6394a69a2529e515f82851c is 50, key is test_row_0/B:col10/1731916427335/Put/seqid=0 2024-11-18T07:53:47,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742204_1380 (size=12301) 2024-11-18T07:53:47,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916427381, max timestamp=1731916427637Num uniques 859; 2024-11-18T07:53:47,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:53:47,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916427381, max timestamp=1731916427637Num uniques 859; 2024-11-18T07:53:47,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:53:47,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916427381, max timestamp=1731916427637Num uniques 859; 2024-11-18T07:53:47,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:53:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T07:53:47,708 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:47,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-18T07:53:47,720 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T07:53:47,727 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:47,727 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T07:53:47,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-18T07:53:47,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:47,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:47,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:47,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:47,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:47,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/3873b390e6394a69a2529e515f82851c 2024-11-18T07:53:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:47,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/fcb073fe992c4b2583fbf13bebb7bbd2 is 50, key is test_row_0/C:col10/1731916427335/Put/seqid=0 2024-11-18T07:53:47,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916487937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:47,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:47,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:47,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:47,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916487948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:47,964 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_2' in 4a6df03ed24f69072d2b4a08bac0a9a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:53:47,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:47,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742205_1381 (size=12301) 2024-11-18T07:53:47,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/fcb073fe992c4b2583fbf13bebb7bbd2 2024-11-18T07:53:47,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/0c6e6efb8f164168bd06710a9181db37 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/0c6e6efb8f164168bd06710a9181db37 2024-11-18T07:53:48,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916487961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:48,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916487963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:48,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/0c6e6efb8f164168bd06710a9181db37, entries=200, sequenceid=296, filesize=39.0 K 2024-11-18T07:53:48,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/3873b390e6394a69a2529e515f82851c as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/3873b390e6394a69a2529e515f82851c 2024-11-18T07:53:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T07:53:48,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/3873b390e6394a69a2529e515f82851c, entries=150, sequenceid=296, filesize=12.0 K 2024-11-18T07:53:48,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/fcb073fe992c4b2583fbf13bebb7bbd2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/fcb073fe992c4b2583fbf13bebb7bbd2 2024-11-18T07:53:48,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/fcb073fe992c4b2583fbf13bebb7bbd2, entries=150, sequenceid=296, filesize=12.0 K 2024-11-18T07:53:48,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 717ms, sequenceid=296, compaction requested=true 2024-11-18T07:53:48,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:48,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:53:48,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:48,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:53:48,055 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:53:48,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:48,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:53:48,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:53:48,055 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:53:48,057 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:53:48,058 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/A is initiating minor compaction (all files) 2024-11-18T07:53:48,058 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/A in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:48,058 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/a706b0a02c3942b8a312a809ebef9bf5, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/63a445af98c64746b9f33b371811f8d6, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/9c7b6e3d98d44261bdb31d6e487b662b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/0c6e6efb8f164168bd06710a9181db37] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=139.2 K 2024-11-18T07:53:48,058 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:48,058 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/a706b0a02c3942b8a312a809ebef9bf5, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/63a445af98c64746b9f33b371811f8d6, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/9c7b6e3d98d44261bdb31d6e487b662b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/0c6e6efb8f164168bd06710a9181db37] 2024-11-18T07:53:48,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-18T07:53:48,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:48,059 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-18T07:53:48,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:48,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:48,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:48,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:48,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:48,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:48,061 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a706b0a02c3942b8a312a809ebef9bf5, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731916423823 2024-11-18T07:53:48,065 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63a445af98c64746b9f33b371811f8d6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731916424641 2024-11-18T07:53:48,068 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c7b6e3d98d44261bdb31d6e487b662b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731916425700 2024-11-18T07:53:48,068 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:53:48,068 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/B is initiating minor compaction (all files) 2024-11-18T07:53:48,068 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/B in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:48,068 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/5c7d2764a4534a53acac7cb81bf8758e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/9d257b625aee4e92910b00f1b0fcf05d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/873f9184bf5b45018c8a8a225eecfad4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/3873b390e6394a69a2529e515f82851c] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=45.9 K 2024-11-18T07:53:48,069 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c6e6efb8f164168bd06710a9181db37, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731916426599 2024-11-18T07:53:48,070 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c7d2764a4534a53acac7cb81bf8758e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731916423823 2024-11-18T07:53:48,070 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d257b625aee4e92910b00f1b0fcf05d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731916425397 2024-11-18T07:53:48,073 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 873f9184bf5b45018c8a8a225eecfad4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731916425700 2024-11-18T07:53:48,076 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3873b390e6394a69a2529e515f82851c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731916426609 2024-11-18T07:53:48,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f0e6af07834a4adc8df12fc6dd4db09a_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916427837/Put/seqid=0 2024-11-18T07:53:48,096 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:48,104 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#B#compaction#320 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:48,104 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/3e461fc7b88c44919f56f02f3390b753 is 50, key is test_row_0/B:col10/1731916427335/Put/seqid=0 2024-11-18T07:53:48,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:48,126 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111861d0dec7258c4b81953e122b2c9a70e5_4a6df03ed24f69072d2b4a08bac0a9a6 store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:48,128 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111861d0dec7258c4b81953e122b2c9a70e5_4a6df03ed24f69072d2b4a08bac0a9a6, store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:48,129 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111861d0dec7258c4b81953e122b2c9a70e5_4a6df03ed24f69072d2b4a08bac0a9a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:48,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742206_1382 (size=12454) 2024-11-18T07:53:48,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:48,208 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f0e6af07834a4adc8df12fc6dd4db09a_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f0e6af07834a4adc8df12fc6dd4db09a_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:48,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/92a0e4d430b44c5493c6e1933901ff30, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:48,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/92a0e4d430b44c5493c6e1933901ff30 is 175, key is test_row_0/A:col10/1731916427837/Put/seqid=0 2024-11-18T07:53:48,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742207_1383 (size=12881) 2024-11-18T07:53:48,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916488189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:48,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916488192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:48,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:48,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916488196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:48,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:48,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:48,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:48,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742208_1384 (size=4469) 2024-11-18T07:53:48,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742209_1385 (size=31255) 2024-11-18T07:53:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T07:53:48,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:48,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916488349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:48,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:48,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916488350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:48,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916488350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:48,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916488588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:48,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916488588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:48,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:48,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916488588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:48,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:48,651 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#A#compaction#319 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:48,652 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/4623407c1fc1417fae174be9723f8b51 is 175, key is test_row_0/A:col10/1731916427335/Put/seqid=0 2024-11-18T07:53:48,653 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/3e461fc7b88c44919f56f02f3390b753 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/3e461fc7b88c44919f56f02f3390b753 2024-11-18T07:53:48,663 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/B of 4a6df03ed24f69072d2b4a08bac0a9a6 into 3e461fc7b88c44919f56f02f3390b753(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:48,663 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:48,663 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/B, priority=12, startTime=1731916428055; duration=0sec 2024-11-18T07:53:48,664 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:48,664 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:B 2024-11-18T07:53:48,664 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:53:48,666 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:53:48,666 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/C is initiating minor compaction (all files) 2024-11-18T07:53:48,666 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/C in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:48,667 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/7c60a93d8e1f4edaafb48569526910c0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/ff3700dcd42349349cb5ac9de3c037b8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/81e926bed7084fd8a32828749c3ae683, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/fcb073fe992c4b2583fbf13bebb7bbd2] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=45.9 K 2024-11-18T07:53:48,667 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c60a93d8e1f4edaafb48569526910c0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731916423823 2024-11-18T07:53:48,668 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting ff3700dcd42349349cb5ac9de3c037b8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731916425397 2024-11-18T07:53:48,668 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 81e926bed7084fd8a32828749c3ae683, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731916425700 2024-11-18T07:53:48,668 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting fcb073fe992c4b2583fbf13bebb7bbd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731916426609 2024-11-18T07:53:48,681 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/92a0e4d430b44c5493c6e1933901ff30 2024-11-18T07:53:48,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/f588c37694d942348779299aae133485 is 50, key is test_row_0/B:col10/1731916427837/Put/seqid=0 2024-11-18T07:53:48,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742210_1386 (size=31835) 2024-11-18T07:53:48,717 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#C#compaction#322 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:48,718 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/af60a9b0c3914107bb59624fe3eb8ba8 is 50, key is test_row_0/C:col10/1731916427335/Put/seqid=0 2024-11-18T07:53:48,735 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/4623407c1fc1417fae174be9723f8b51 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/4623407c1fc1417fae174be9723f8b51 2024-11-18T07:53:48,746 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/A of 4a6df03ed24f69072d2b4a08bac0a9a6 into 4623407c1fc1417fae174be9723f8b51(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:48,746 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:48,746 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/A, priority=12, startTime=1731916428055; duration=0sec 2024-11-18T07:53:48,746 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:48,746 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:A 2024-11-18T07:53:48,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742211_1387 (size=12301) 2024-11-18T07:53:48,775 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/f588c37694d942348779299aae133485 2024-11-18T07:53:48,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742212_1388 (size=12881) 2024-11-18T07:53:48,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/d8959ebb7eb94058a8ed15cf7e90ff7c is 50, key is test_row_0/C:col10/1731916427837/Put/seqid=0 2024-11-18T07:53:48,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742213_1389 (size=12301) 2024-11-18T07:53:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T07:53:48,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55020 deadline: 1731916488890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:48,906 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_2' in 4a6df03ed24f69072d2b4a08bac0a9a6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:53:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:48,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916488928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:48,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916488928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:48,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916488929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:49,181 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/af60a9b0c3914107bb59624fe3eb8ba8 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/af60a9b0c3914107bb59624fe3eb8ba8 2024-11-18T07:53:49,191 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/C of 4a6df03ed24f69072d2b4a08bac0a9a6 into af60a9b0c3914107bb59624fe3eb8ba8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:49,191 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:49,191 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/C, priority=12, startTime=1731916428055; duration=0sec 2024-11-18T07:53:49,192 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:49,192 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:C 2024-11-18T07:53:49,253 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/d8959ebb7eb94058a8ed15cf7e90ff7c 2024-11-18T07:53:49,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/92a0e4d430b44c5493c6e1933901ff30 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/92a0e4d430b44c5493c6e1933901ff30 2024-11-18T07:53:49,263 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/92a0e4d430b44c5493c6e1933901ff30, entries=150, sequenceid=333, filesize=30.5 K 2024-11-18T07:53:49,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/f588c37694d942348779299aae133485 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/f588c37694d942348779299aae133485 2024-11-18T07:53:49,269 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/f588c37694d942348779299aae133485, entries=150, sequenceid=333, filesize=12.0 K 2024-11-18T07:53:49,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/d8959ebb7eb94058a8ed15cf7e90ff7c as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/d8959ebb7eb94058a8ed15cf7e90ff7c 2024-11-18T07:53:49,294 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/d8959ebb7eb94058a8ed15cf7e90ff7c, entries=150, sequenceid=333, filesize=12.0 K 2024-11-18T07:53:49,296 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 1237ms, sequenceid=333, compaction requested=false 2024-11-18T07:53:49,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:49,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:49,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-18T07:53:49,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-18T07:53:49,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-18T07:53:49,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5750 sec 2024-11-18T07:53:49,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.5990 sec 2024-11-18T07:53:49,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T07:53:49,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:49,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:49,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:49,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:49,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:49,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:49,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185c2ad833c98946ce9c12605c3b19d27d_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916429470/Put/seqid=0 2024-11-18T07:53:49,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742214_1390 (size=12454) 2024-11-18T07:53:49,533 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:49,538 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185c2ad833c98946ce9c12605c3b19d27d_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185c2ad833c98946ce9c12605c3b19d27d_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:49,539 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/a31caf38d91f429d996f5c8b1ce52d03, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:49,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/a31caf38d91f429d996f5c8b1ce52d03 is 175, key is test_row_0/A:col10/1731916429470/Put/seqid=0 2024-11-18T07:53:49,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742215_1391 (size=31255) 2024-11-18T07:53:49,588 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/a31caf38d91f429d996f5c8b1ce52d03 2024-11-18T07:53:49,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/ca0184765e8a47a3a091d9cc1912a523 is 50, key is test_row_0/B:col10/1731916429470/Put/seqid=0 2024-11-18T07:53:49,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916489623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:49,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916489650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:49,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:49,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:49,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:49,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916489650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:49,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:49,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:49,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:49,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742216_1392 (size=12301) 2024-11-18T07:53:49,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/ca0184765e8a47a3a091d9cc1912a523 2024-11-18T07:53:49,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916489758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:49,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916489778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:49,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916489778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:49,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:49,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:49,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:49,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/94e4e476076447018af2fbf81e3404e7 is 50, key is test_row_0/C:col10/1731916429470/Put/seqid=0 2024-11-18T07:53:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T07:53:49,868 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:49,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:49,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742217_1393 (size=12301) 2024-11-18T07:53:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-18T07:53:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T07:53:49,879 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:49,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/94e4e476076447018af2fbf81e3404e7 2024-11-18T07:53:49,882 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:49,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:49,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/a31caf38d91f429d996f5c8b1ce52d03 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/a31caf38d91f429d996f5c8b1ce52d03 2024-11-18T07:53:49,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/a31caf38d91f429d996f5c8b1ce52d03, entries=150, sequenceid=348, filesize=30.5 K 2024-11-18T07:53:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T07:53:49,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/ca0184765e8a47a3a091d9cc1912a523 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/ca0184765e8a47a3a091d9cc1912a523 2024-11-18T07:53:50,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731916489978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:50,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916489989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:50,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:50,012 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_2' in 4a6df03ed24f69072d2b4a08bac0a9a6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4298 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:53:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:50,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916489999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:50,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916489999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:50,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T07:53:50,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:50,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:50,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/ca0184765e8a47a3a091d9cc1912a523, entries=150, sequenceid=348, filesize=12.0 K 2024-11-18T07:53:50,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/94e4e476076447018af2fbf81e3404e7 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/94e4e476076447018af2fbf81e3404e7 2024-11-18T07:53:50,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/94e4e476076447018af2fbf81e3404e7, entries=150, sequenceid=348, filesize=12.0 K 2024-11-18T07:53:50,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 606ms, sequenceid=348, compaction requested=true 2024-11-18T07:53:50,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:50,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:53:50,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:50,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:53:50,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:53:50,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:53:50,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T07:53:50,081 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:50,081 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:50,091 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:50,091 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/A is initiating minor compaction (all files) 2024-11-18T07:53:50,091 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/A in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:50,091 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/4623407c1fc1417fae174be9723f8b51, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/92a0e4d430b44c5493c6e1933901ff30, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/a31caf38d91f429d996f5c8b1ce52d03] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=92.1 K 2024-11-18T07:53:50,091 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:50,091 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/4623407c1fc1417fae174be9723f8b51, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/92a0e4d430b44c5493c6e1933901ff30, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/a31caf38d91f429d996f5c8b1ce52d03] 2024-11-18T07:53:50,092 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:50,092 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/C is initiating minor compaction (all files) 2024-11-18T07:53:50,092 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/C in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:50,093 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/af60a9b0c3914107bb59624fe3eb8ba8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/d8959ebb7eb94058a8ed15cf7e90ff7c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/94e4e476076447018af2fbf81e3404e7] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=36.6 K 2024-11-18T07:53:50,095 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4623407c1fc1417fae174be9723f8b51, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731916426609 2024-11-18T07:53:50,095 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting af60a9b0c3914107bb59624fe3eb8ba8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731916426609 2024-11-18T07:53:50,099 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting d8959ebb7eb94058a8ed15cf7e90ff7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731916427824 2024-11-18T07:53:50,099 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 94e4e476076447018af2fbf81e3404e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731916428182 2024-11-18T07:53:50,103 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92a0e4d430b44c5493c6e1933901ff30, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731916427824 2024-11-18T07:53:50,103 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a31caf38d91f429d996f5c8b1ce52d03, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731916428182 2024-11-18T07:53:50,125 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#C#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:50,125 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/26809c138efd4a73984ec03662d36f02 is 50, key is test_row_0/C:col10/1731916429470/Put/seqid=0 2024-11-18T07:53:50,128 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:50,131 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411186f177a84bf714ed1a867d09d1cc1d988_4a6df03ed24f69072d2b4a08bac0a9a6 store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:50,133 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411186f177a84bf714ed1a867d09d1cc1d988_4a6df03ed24f69072d2b4a08bac0a9a6, store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:50,133 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186f177a84bf714ed1a867d09d1cc1d988_4a6df03ed24f69072d2b4a08bac0a9a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:50,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T07:53:50,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T07:53:50,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:50,204 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T07:53:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:50,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742218_1394 (size=12983) 2024-11-18T07:53:50,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742219_1395 (size=4469) 2024-11-18T07:53:50,249 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/26809c138efd4a73984ec03662d36f02 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/26809c138efd4a73984ec03662d36f02 2024-11-18T07:53:50,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185ca38f9a215546b9b83558fba3347a7f_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916429615/Put/seqid=0 2024-11-18T07:53:50,263 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/C of 4a6df03ed24f69072d2b4a08bac0a9a6 into 26809c138efd4a73984ec03662d36f02(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:50,263 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:50,263 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/C, priority=13, startTime=1731916430081; duration=0sec 2024-11-18T07:53:50,264 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:50,264 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:C 2024-11-18T07:53:50,264 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:50,268 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:50,268 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/B is initiating minor compaction (all files) 2024-11-18T07:53:50,268 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/B in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:50,268 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/3e461fc7b88c44919f56f02f3390b753, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/f588c37694d942348779299aae133485, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/ca0184765e8a47a3a091d9cc1912a523] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=36.6 K 2024-11-18T07:53:50,269 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e461fc7b88c44919f56f02f3390b753, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731916426609 2024-11-18T07:53:50,269 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting f588c37694d942348779299aae133485, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731916427824 2024-11-18T07:53:50,269 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting ca0184765e8a47a3a091d9cc1912a523, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731916428182 2024-11-18T07:53:50,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742220_1396 (size=12454) 2024-11-18T07:53:50,300 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#B#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:50,301 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/0dc3d6d305cc4c94bc8294c64f54a488 is 50, key is test_row_0/B:col10/1731916429470/Put/seqid=0 2024-11-18T07:53:50,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. as already flushing 2024-11-18T07:53:50,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:50,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742221_1397 (size=12983) 2024-11-18T07:53:50,363 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/0dc3d6d305cc4c94bc8294c64f54a488 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/0dc3d6d305cc4c94bc8294c64f54a488 2024-11-18T07:53:50,394 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/B of 4a6df03ed24f69072d2b4a08bac0a9a6 into 0dc3d6d305cc4c94bc8294c64f54a488(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:50,394 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:50,394 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/B, priority=13, startTime=1731916430080; duration=0sec 2024-11-18T07:53:50,394 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:50,394 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:B 2024-11-18T07:53:50,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:50,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916490397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:50,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:50,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:50,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916490409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:50,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:50,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916490411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T07:53:50,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:50,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916490530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:50,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:50,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916490538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:50,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:50,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916490538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:50,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:50,635 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#A#compaction#328 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:50,636 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/5544d31440744d87a320187af352a2c6 is 175, key is test_row_0/A:col10/1731916429470/Put/seqid=0 2024-11-18T07:53:50,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:50,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742222_1398 (size=31937) 2024-11-18T07:53:50,734 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185ca38f9a215546b9b83558fba3347a7f_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185ca38f9a215546b9b83558fba3347a7f_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:50,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/7acbf35b6e1045a085863d5d6db8cd33, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:50,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/7acbf35b6e1045a085863d5d6db8cd33 is 175, key is test_row_0/A:col10/1731916429615/Put/seqid=0 2024-11-18T07:53:50,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916490739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:50,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742223_1399 (size=31255) 2024-11-18T07:53:50,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916490768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:50,783 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/7acbf35b6e1045a085863d5d6db8cd33 2024-11-18T07:53:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:50,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916490771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:50,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:50,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/8712609693274199bbbeb8a540ddf999 is 50, key is test_row_0/B:col10/1731916429615/Put/seqid=0 2024-11-18T07:53:50,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742224_1400 (size=12301) 2024-11-18T07:53:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T07:53:51,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55012 deadline: 1731916491068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:51,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:51,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:51,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:51,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731916491088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:51,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:51,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:51,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:51,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55034 deadline: 1731916491099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:53:51,145 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/5544d31440744d87a320187af352a2c6 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/5544d31440744d87a320187af352a2c6 2024-11-18T07:53:51,151 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/A of 4a6df03ed24f69072d2b4a08bac0a9a6 into 5544d31440744d87a320187af352a2c6(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-18T07:53:51,151 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:51,151 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/A, priority=13, startTime=1731916430080; duration=1sec 2024-11-18T07:53:51,151 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:51,151 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:A 2024-11-18T07:53:51,323 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/8712609693274199bbbeb8a540ddf999 2024-11-18T07:53:51,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/3711a8ce3c0546f0ab8e1a7e24c103cb is 50, key is test_row_0/C:col10/1731916429615/Put/seqid=0 2024-11-18T07:53:51,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742225_1401 (size=12301) 2024-11-18T07:53:51,392 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/3711a8ce3c0546f0ab8e1a7e24c103cb 2024-11-18T07:53:51,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/7acbf35b6e1045a085863d5d6db8cd33 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/7acbf35b6e1045a085863d5d6db8cd33 2024-11-18T07:53:51,475 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/7acbf35b6e1045a085863d5d6db8cd33, entries=150, sequenceid=368, filesize=30.5 K 2024-11-18T07:53:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/8712609693274199bbbeb8a540ddf999 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/8712609693274199bbbeb8a540ddf999 2024-11-18T07:53:51,488 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/8712609693274199bbbeb8a540ddf999, entries=150, sequenceid=368, filesize=12.0 K 2024-11-18T07:53:51,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/3711a8ce3c0546f0ab8e1a7e24c103cb as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/3711a8ce3c0546f0ab8e1a7e24c103cb 2024-11-18T07:53:51,495 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/3711a8ce3c0546f0ab8e1a7e24c103cb, entries=150, sequenceid=368, filesize=12.0 K 2024-11-18T07:53:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,497 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 1293ms, sequenceid=368, compaction requested=false 2024-11-18T07:53:51,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:51,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:51,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-18T07:53:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-18T07:53:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,512 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-18T07:53:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,512 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6160 sec 2024-11-18T07:53:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.6440 sec 2024-11-18T07:53:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T07:53:51,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:51,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:51,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:51,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:51,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:51,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118925877c6e20349808478eeff0ba39136_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_1/A:col10/1731916431634/Put/seqid=0 2024-11-18T07:53:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742226_1402 (size=12454) 2024-11-18T07:53:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,694 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,703 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118925877c6e20349808478eeff0ba39136_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118925877c6e20349808478eeff0ba39136_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,705 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/55fff3d259f24c5c845dfe768df96f69, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/55fff3d259f24c5c845dfe768df96f69 is 175, key is test_row_1/A:col10/1731916431634/Put/seqid=0 2024-11-18T07:53:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,709 INFO [Thread-1458 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1458. 2024-11-18T07:53:51,709 DEBUG [Thread-1458 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:51,709 DEBUG [Thread-1458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:51,709 DEBUG [Thread-1458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,710 INFO [Thread-1441 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1441. 2024-11-18T07:53:51,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:51,710 DEBUG [Thread-1441 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:51,710 DEBUG [Thread-1441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:51,710 DEBUG [Thread-1441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:51,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,726 INFO [Thread-1450 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1450. 2024-11-18T07:53:51,726 DEBUG [Thread-1450 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:51,726 DEBUG [Thread-1450 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:51,726 DEBUG [Thread-1450 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:51,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,733 INFO [Thread-1454 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1454. 2024-11-18T07:53:51,733 DEBUG [Thread-1454 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:51,733 DEBUG [Thread-1454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:51,734 DEBUG [Thread-1454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,737 INFO [Thread-1445 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1445. 2024-11-18T07:53:51,737 DEBUG [Thread-1445 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:51,737 DEBUG [Thread-1445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:51,738 DEBUG [Thread-1445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:51,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,743 INFO [Thread-1447 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1447. 2024-11-18T07:53:51,743 DEBUG [Thread-1447 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:51,743 DEBUG [Thread-1447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:51,743 DEBUG [Thread-1447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:51,745 INFO [Thread-1456 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1456. 2024-11-18T07:53:51,745 DEBUG [Thread-1456 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:51,745 DEBUG [Thread-1456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:51,745 DEBUG [Thread-1456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:51,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:51,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:51,761 INFO [Thread-1452 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1452. 2024-11-18T07:53:51,761 DEBUG [Thread-1452 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:51,761 DEBUG [Thread-1452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:51,761 DEBUG [Thread-1452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:51,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742227_1403 (size=31251) 2024-11-18T07:53:51,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:51,766 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/55fff3d259f24c5c845dfe768df96f69 2024-11-18T07:53:51,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/2ad0b3b934824e3f8071be7b6b2325d8 is 50, key is test_row_1/B:col10/1731916431634/Put/seqid=0 2024-11-18T07:53:51,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742228_1404 (size=9857) 2024-11-18T07:53:51,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/2ad0b3b934824e3f8071be7b6b2325d8 2024-11-18T07:53:51,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/28e06b233d4c42d08a25d8617230015d is 50, key is test_row_1/C:col10/1731916431634/Put/seqid=0 2024-11-18T07:53:51,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742229_1405 (size=9857) 2024-11-18T07:53:51,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/28e06b233d4c42d08a25d8617230015d 2024-11-18T07:53:51,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/55fff3d259f24c5c845dfe768df96f69 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/55fff3d259f24c5c845dfe768df96f69 2024-11-18T07:53:51,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/55fff3d259f24c5c845dfe768df96f69, entries=150, sequenceid=384, filesize=30.5 K 2024-11-18T07:53:51,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/2ad0b3b934824e3f8071be7b6b2325d8 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/2ad0b3b934824e3f8071be7b6b2325d8 2024-11-18T07:53:51,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/2ad0b3b934824e3f8071be7b6b2325d8, entries=100, sequenceid=384, filesize=9.6 K 2024-11-18T07:53:51,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/28e06b233d4c42d08a25d8617230015d as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/28e06b233d4c42d08a25d8617230015d 2024-11-18T07:53:51,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/28e06b233d4c42d08a25d8617230015d, entries=100, sequenceid=384, filesize=9.6 K 2024-11-18T07:53:51,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=33.54 KB/34350 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 223ms, sequenceid=384, compaction requested=true 2024-11-18T07:53:51,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:51,861 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:51,862 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:51,862 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/A is initiating minor compaction (all files) 2024-11-18T07:53:51,863 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/A in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:51,863 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/5544d31440744d87a320187af352a2c6, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/7acbf35b6e1045a085863d5d6db8cd33, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/55fff3d259f24c5c845dfe768df96f69] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=92.2 K 2024-11-18T07:53:51,863 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:51,863 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/5544d31440744d87a320187af352a2c6, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/7acbf35b6e1045a085863d5d6db8cd33, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/55fff3d259f24c5c845dfe768df96f69] 2024-11-18T07:53:51,864 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5544d31440744d87a320187af352a2c6, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731916428182 2024-11-18T07:53:51,864 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7acbf35b6e1045a085863d5d6db8cd33, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731916429615 2024-11-18T07:53:51,869 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55fff3d259f24c5c845dfe768df96f69, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731916430402 2024-11-18T07:53:51,881 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:51,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:53:51,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:51,883 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:51,884 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411186e12ea9f1abe4880aefb35a597000486_4a6df03ed24f69072d2b4a08bac0a9a6 store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:51,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:53:51,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:51,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a6df03ed24f69072d2b4a08bac0a9a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:53:51,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:51,893 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411186e12ea9f1abe4880aefb35a597000486_4a6df03ed24f69072d2b4a08bac0a9a6, store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:51,894 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:51,894 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/B is initiating minor compaction (all files) 2024-11-18T07:53:51,894 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/B in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:51,894 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/0dc3d6d305cc4c94bc8294c64f54a488, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/8712609693274199bbbeb8a540ddf999, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/2ad0b3b934824e3f8071be7b6b2325d8] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=34.3 K 2024-11-18T07:53:51,897 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dc3d6d305cc4c94bc8294c64f54a488, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731916428182 2024-11-18T07:53:51,898 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8712609693274199bbbeb8a540ddf999, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731916429615 2024-11-18T07:53:51,898 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ad0b3b934824e3f8071be7b6b2325d8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731916430409 2024-11-18T07:53:51,901 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186e12ea9f1abe4880aefb35a597000486_4a6df03ed24f69072d2b4a08bac0a9a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:51,911 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#B#compaction#337 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:51,912 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/90626b6f80c343a4911203db2559b8af is 50, key is test_row_0/B:col10/1731916429615/Put/seqid=0 2024-11-18T07:53:51,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742231_1407 (size=13085) 2024-11-18T07:53:51,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742230_1406 (size=4469) 2024-11-18T07:53:51,940 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/90626b6f80c343a4911203db2559b8af as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/90626b6f80c343a4911203db2559b8af 2024-11-18T07:53:51,948 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/B of 4a6df03ed24f69072d2b4a08bac0a9a6 into 90626b6f80c343a4911203db2559b8af(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:51,948 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:51,948 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/B, priority=13, startTime=1731916431883; duration=0sec 2024-11-18T07:53:51,948 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:53:51,948 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:B 2024-11-18T07:53:51,948 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:53:51,949 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#A#compaction#336 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:51,949 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:53:51,949 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): 4a6df03ed24f69072d2b4a08bac0a9a6/C is initiating minor compaction (all files) 2024-11-18T07:53:51,950 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a6df03ed24f69072d2b4a08bac0a9a6/C in TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:51,950 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/a564e6baa9784042baf7ef8de9b83b32 is 175, key is test_row_0/A:col10/1731916429615/Put/seqid=0 2024-11-18T07:53:51,950 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/26809c138efd4a73984ec03662d36f02, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/3711a8ce3c0546f0ab8e1a7e24c103cb, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/28e06b233d4c42d08a25d8617230015d] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp, totalSize=34.3 K 2024-11-18T07:53:51,952 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 26809c138efd4a73984ec03662d36f02, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731916428182 2024-11-18T07:53:51,952 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3711a8ce3c0546f0ab8e1a7e24c103cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731916429615 2024-11-18T07:53:51,952 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 28e06b233d4c42d08a25d8617230015d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731916430409 2024-11-18T07:53:51,965 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a6df03ed24f69072d2b4a08bac0a9a6#C#compaction#338 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:53:51,965 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/f7c12361430f471b9cec6d84fbf9d076 is 50, key is test_row_0/C:col10/1731916429615/Put/seqid=0 2024-11-18T07:53:51,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742232_1408 (size=32146) 2024-11-18T07:53:51,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742233_1409 (size=13085) 2024-11-18T07:53:51,984 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/f7c12361430f471b9cec6d84fbf9d076 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/f7c12361430f471b9cec6d84fbf9d076 2024-11-18T07:53:51,991 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/C of 4a6df03ed24f69072d2b4a08bac0a9a6 into f7c12361430f471b9cec6d84fbf9d076(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:51,991 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:51,991 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/C, priority=13, startTime=1731916431884; duration=0sec 2024-11-18T07:53:51,991 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:51,991 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:C 2024-11-18T07:53:51,995 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/a564e6baa9784042baf7ef8de9b83b32 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/a564e6baa9784042baf7ef8de9b83b32 2024-11-18T07:53:52,001 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a6df03ed24f69072d2b4a08bac0a9a6/A of 4a6df03ed24f69072d2b4a08bac0a9a6 into a564e6baa9784042baf7ef8de9b83b32(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:53:52,001 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:52,001 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6., storeName=4a6df03ed24f69072d2b4a08bac0a9a6/A, priority=13, startTime=1731916431861; duration=0sec 2024-11-18T07:53:52,001 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:53:52,001 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a6df03ed24f69072d2b4a08bac0a9a6:A 2024-11-18T07:53:52,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T07:53:52,027 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:52,943 INFO [Thread-1443 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1443. 2024-11-18T07:53:52,943 DEBUG [Thread-1443 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:52,943 DEBUG [Thread-1443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:52,943 DEBUG [Thread-1443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:52,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:54,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a6df03ed24f69072d2b4a08bac0a9a6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:53:54,051 INFO [Thread-1439 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1439. 2024-11-18T07:53:54,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=A 2024-11-18T07:53:54,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:54,051 DEBUG [Thread-1439 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:53:54,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=B 2024-11-18T07:53:54,051 DEBUG [Thread-1439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:54,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:54,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a6df03ed24f69072d2b4a08bac0a9a6, store=C 2024-11-18T07:53:54,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:54,051 DEBUG [Thread-1439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:54,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 540 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1603 rows 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 534 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1601 rows 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 538 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1601 rows 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 533 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1589 rows 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 540 2024-11-18T07:53:54,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1620 rows 2024-11-18T07:53:54,052 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T07:53:54,052 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T07:53:54,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:54,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:54,053 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:53:54,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T07:53:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T07:53:54,066 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916434065"}]},"ts":"1731916434065"} 2024-11-18T07:53:54,069 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T07:53:54,069 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T07:53:54,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T07:53:54,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188408631c2b2c4c3989ea46b89f65281b_4a6df03ed24f69072d2b4a08bac0a9a6 is 50, key is test_row_0/A:col10/1731916431706/Put/seqid=0 2024-11-18T07:53:54,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a6df03ed24f69072d2b4a08bac0a9a6, UNASSIGN}] 2024-11-18T07:53:54,076 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a6df03ed24f69072d2b4a08bac0a9a6, UNASSIGN 2024-11-18T07:53:54,077 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=4a6df03ed24f69072d2b4a08bac0a9a6, regionState=CLOSING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:53:54,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742234_1410 (size=12454) 2024-11-18T07:53:54,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a6df03ed24f69072d2b4a08bac0a9a6, UNASSIGN because future has completed 2024-11-18T07:53:54,080 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T07:53:54,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:53:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T07:53:54,235 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(122): Close 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:54,235 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T07:53:54,235 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1722): Closing 4a6df03ed24f69072d2b4a08bac0a9a6, disabling compactions & flushes 2024-11-18T07:53:54,235 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T07:53:54,480 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:54,486 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188408631c2b2c4c3989ea46b89f65281b_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188408631c2b2c4c3989ea46b89f65281b_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:54,487 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/51eda9a2014948018f494ba4e75b23e0, store: [table=TestAcidGuarantees family=A region=4a6df03ed24f69072d2b4a08bac0a9a6] 2024-11-18T07:53:54,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/51eda9a2014948018f494ba4e75b23e0 is 175, key is test_row_0/A:col10/1731916431706/Put/seqid=0 2024-11-18T07:53:54,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742235_1411 (size=31255) 2024-11-18T07:53:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T07:53:54,893 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=397, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/51eda9a2014948018f494ba4e75b23e0 2024-11-18T07:53:54,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/47cfd3b8bc2a4da2a7fc68e57f9a7470 is 50, key is test_row_0/B:col10/1731916431706/Put/seqid=0 2024-11-18T07:53:54,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742236_1412 (size=12301) 2024-11-18T07:53:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T07:53:55,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/47cfd3b8bc2a4da2a7fc68e57f9a7470 2024-11-18T07:53:55,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/3f2a3748e1974827a38b8cf819d8bcd3 is 50, key is test_row_0/C:col10/1731916431706/Put/seqid=0 2024-11-18T07:53:55,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742237_1413 (size=12301) 2024-11-18T07:53:55,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/3f2a3748e1974827a38b8cf819d8bcd3 2024-11-18T07:53:55,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/A/51eda9a2014948018f494ba4e75b23e0 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/51eda9a2014948018f494ba4e75b23e0 2024-11-18T07:53:55,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/51eda9a2014948018f494ba4e75b23e0, entries=150, sequenceid=397, filesize=30.5 K 2024-11-18T07:53:55,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/B/47cfd3b8bc2a4da2a7fc68e57f9a7470 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/47cfd3b8bc2a4da2a7fc68e57f9a7470 2024-11-18T07:53:55,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/47cfd3b8bc2a4da2a7fc68e57f9a7470, entries=150, sequenceid=397, filesize=12.0 K 2024-11-18T07:53:55,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/.tmp/C/3f2a3748e1974827a38b8cf819d8bcd3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/3f2a3748e1974827a38b8cf819d8bcd3 2024-11-18T07:53:55,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/3f2a3748e1974827a38b8cf819d8bcd3, entries=150, sequenceid=397, filesize=12.0 K 2024-11-18T07:53:55,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 4a6df03ed24f69072d2b4a08bac0a9a6 in 1293ms, sequenceid=397, compaction requested=false 2024-11-18T07:53:55,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a6df03ed24f69072d2b4a08bac0a9a6: 2024-11-18T07:53:55,343 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:55,343 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:55,344 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. after waiting 0 ms 2024-11-18T07:53:55,344 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:55,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/17372032f64d46e69ab3a52d5f6441c0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/95410d9d1a1241a5b30dbbca05789d93, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/081ca9039ec7489084e8e507326deb37, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/cb29d4d8f7fd4905820494fa01639b5e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/6f2098d967b14870b89a029fc648402f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/c46b882c2eee42c293463636c6701dfc, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/41a14a2d590f48d496e831d13365a396, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/cb07c9f98e404bbc803db2711b53e29a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/91af8157a6bb4c24ab0f783710ba6644, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/5f6152fbe99b49819deb6ee00d3c9154, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/479da5a9af694584beddd79245c5fe07, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/97b904922f944fc0b572fafa1037b4c7, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/b0dc4fb934ae475687e83104d134f742, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/a706b0a02c3942b8a312a809ebef9bf5, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/63a445af98c64746b9f33b371811f8d6, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/9c7b6e3d98d44261bdb31d6e487b662b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/0c6e6efb8f164168bd06710a9181db37, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/4623407c1fc1417fae174be9723f8b51, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/92a0e4d430b44c5493c6e1933901ff30, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/5544d31440744d87a320187af352a2c6, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/a31caf38d91f429d996f5c8b1ce52d03, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/7acbf35b6e1045a085863d5d6db8cd33, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/55fff3d259f24c5c845dfe768df96f69] to archive 2024-11-18T07:53:55,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:53:55,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/17372032f64d46e69ab3a52d5f6441c0 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/17372032f64d46e69ab3a52d5f6441c0 2024-11-18T07:53:55,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/95410d9d1a1241a5b30dbbca05789d93 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/95410d9d1a1241a5b30dbbca05789d93 2024-11-18T07:53:55,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/081ca9039ec7489084e8e507326deb37 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/081ca9039ec7489084e8e507326deb37 2024-11-18T07:53:55,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/cb29d4d8f7fd4905820494fa01639b5e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/cb29d4d8f7fd4905820494fa01639b5e 2024-11-18T07:53:55,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/6f2098d967b14870b89a029fc648402f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/6f2098d967b14870b89a029fc648402f 2024-11-18T07:53:55,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/c46b882c2eee42c293463636c6701dfc to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/c46b882c2eee42c293463636c6701dfc 2024-11-18T07:53:55,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/41a14a2d590f48d496e831d13365a396 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/41a14a2d590f48d496e831d13365a396 2024-11-18T07:53:55,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/cb07c9f98e404bbc803db2711b53e29a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/cb07c9f98e404bbc803db2711b53e29a 2024-11-18T07:53:55,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/91af8157a6bb4c24ab0f783710ba6644 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/91af8157a6bb4c24ab0f783710ba6644 2024-11-18T07:53:55,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/5f6152fbe99b49819deb6ee00d3c9154 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/5f6152fbe99b49819deb6ee00d3c9154 2024-11-18T07:53:55,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/479da5a9af694584beddd79245c5fe07 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/479da5a9af694584beddd79245c5fe07 2024-11-18T07:53:55,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/97b904922f944fc0b572fafa1037b4c7 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/97b904922f944fc0b572fafa1037b4c7 2024-11-18T07:53:55,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/b0dc4fb934ae475687e83104d134f742 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/b0dc4fb934ae475687e83104d134f742 2024-11-18T07:53:55,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/a706b0a02c3942b8a312a809ebef9bf5 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/a706b0a02c3942b8a312a809ebef9bf5 2024-11-18T07:53:55,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/63a445af98c64746b9f33b371811f8d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/63a445af98c64746b9f33b371811f8d6 2024-11-18T07:53:55,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/9c7b6e3d98d44261bdb31d6e487b662b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/9c7b6e3d98d44261bdb31d6e487b662b 2024-11-18T07:53:55,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/0c6e6efb8f164168bd06710a9181db37 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/0c6e6efb8f164168bd06710a9181db37 2024-11-18T07:53:55,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/4623407c1fc1417fae174be9723f8b51 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/4623407c1fc1417fae174be9723f8b51 2024-11-18T07:53:55,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/92a0e4d430b44c5493c6e1933901ff30 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/92a0e4d430b44c5493c6e1933901ff30 2024-11-18T07:53:55,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/5544d31440744d87a320187af352a2c6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/5544d31440744d87a320187af352a2c6 2024-11-18T07:53:55,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/a31caf38d91f429d996f5c8b1ce52d03 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/a31caf38d91f429d996f5c8b1ce52d03 2024-11-18T07:53:55,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/7acbf35b6e1045a085863d5d6db8cd33 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/7acbf35b6e1045a085863d5d6db8cd33 2024-11-18T07:53:55,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/55fff3d259f24c5c845dfe768df96f69 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/55fff3d259f24c5c845dfe768df96f69 2024-11-18T07:53:55,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/ddac30d039f541219759ebcc9ea85a52, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/38140b72c77440f38535e1d2989722c3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/11ab198a16e14a8595086235cb712716, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/b671f3095e4348a1a86fb29e6e37caa9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/e90cecf607ea4328a023f4ffeb158de2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/db004edace994088924838423d8a494e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/776e4fa444ac49aa88f99e56a038e680, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1265f83e0a714cffb9b0130f16189242, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1683bd52dcab44b79e159175ff007b6a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/b94685e1ee6845e2bb4481b8325129ba, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1e488c11ddde43ec8a16c886160f8076, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/25b9f7870e86410e942bee06f97e85bc, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/5c7d2764a4534a53acac7cb81bf8758e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/81f96b6220d94268b53cbadea38785f3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/9d257b625aee4e92910b00f1b0fcf05d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/873f9184bf5b45018c8a8a225eecfad4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/3e461fc7b88c44919f56f02f3390b753, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/3873b390e6394a69a2529e515f82851c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/f588c37694d942348779299aae133485, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/0dc3d6d305cc4c94bc8294c64f54a488, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/ca0184765e8a47a3a091d9cc1912a523, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/8712609693274199bbbeb8a540ddf999, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/2ad0b3b934824e3f8071be7b6b2325d8] to archive 2024-11-18T07:53:55,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:53:55,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/ddac30d039f541219759ebcc9ea85a52 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/ddac30d039f541219759ebcc9ea85a52 2024-11-18T07:53:55,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/38140b72c77440f38535e1d2989722c3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/38140b72c77440f38535e1d2989722c3 2024-11-18T07:53:55,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/11ab198a16e14a8595086235cb712716 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/11ab198a16e14a8595086235cb712716 2024-11-18T07:53:55,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/b671f3095e4348a1a86fb29e6e37caa9 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/b671f3095e4348a1a86fb29e6e37caa9 2024-11-18T07:53:55,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/e90cecf607ea4328a023f4ffeb158de2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/e90cecf607ea4328a023f4ffeb158de2 2024-11-18T07:53:55,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/db004edace994088924838423d8a494e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/db004edace994088924838423d8a494e 2024-11-18T07:53:55,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/776e4fa444ac49aa88f99e56a038e680 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/776e4fa444ac49aa88f99e56a038e680 2024-11-18T07:53:55,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1265f83e0a714cffb9b0130f16189242 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1265f83e0a714cffb9b0130f16189242 2024-11-18T07:53:55,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1683bd52dcab44b79e159175ff007b6a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1683bd52dcab44b79e159175ff007b6a 2024-11-18T07:53:55,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/b94685e1ee6845e2bb4481b8325129ba to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/b94685e1ee6845e2bb4481b8325129ba 2024-11-18T07:53:55,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1e488c11ddde43ec8a16c886160f8076 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/1e488c11ddde43ec8a16c886160f8076 2024-11-18T07:53:55,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/25b9f7870e86410e942bee06f97e85bc to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/25b9f7870e86410e942bee06f97e85bc 2024-11-18T07:53:55,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/5c7d2764a4534a53acac7cb81bf8758e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/5c7d2764a4534a53acac7cb81bf8758e 2024-11-18T07:53:55,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/81f96b6220d94268b53cbadea38785f3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/81f96b6220d94268b53cbadea38785f3 2024-11-18T07:53:55,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/9d257b625aee4e92910b00f1b0fcf05d to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/9d257b625aee4e92910b00f1b0fcf05d 2024-11-18T07:53:55,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/873f9184bf5b45018c8a8a225eecfad4 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/873f9184bf5b45018c8a8a225eecfad4 2024-11-18T07:53:55,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/3e461fc7b88c44919f56f02f3390b753 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/3e461fc7b88c44919f56f02f3390b753 2024-11-18T07:53:55,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/3873b390e6394a69a2529e515f82851c to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/3873b390e6394a69a2529e515f82851c 2024-11-18T07:53:55,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/f588c37694d942348779299aae133485 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/f588c37694d942348779299aae133485 2024-11-18T07:53:55,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/0dc3d6d305cc4c94bc8294c64f54a488 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/0dc3d6d305cc4c94bc8294c64f54a488 2024-11-18T07:53:55,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/ca0184765e8a47a3a091d9cc1912a523 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/ca0184765e8a47a3a091d9cc1912a523 2024-11-18T07:53:55,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/8712609693274199bbbeb8a540ddf999 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/8712609693274199bbbeb8a540ddf999 2024-11-18T07:53:55,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/2ad0b3b934824e3f8071be7b6b2325d8 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/2ad0b3b934824e3f8071be7b6b2325d8 2024-11-18T07:53:55,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/495bd213ab494c38b73b71f01be6fd9f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/b16dbc85b82545a0a7434b73e032394f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/8c73c67cedfd4b08938dcd9d6cd5d874, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/5bc955c704d14e08a6d5b86deb15f3c4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/7f59dddd10964800b9c6f4abd02cafa8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/76439e38243c4f2d833c4ee4059fa3c3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/c9abc1b2a47c48bdafd54355985403fc, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/678e5430fceb433693761ee35e213db5, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/537af44619704fd39c7729f0e8cebfb8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/f5c4fa0a637c4a138f4abb90a8e8a53d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/4b9134b8cab64419ba910a92e1c8e853, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/53be7f525eac4e09b9fb096c62748851, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/7c60a93d8e1f4edaafb48569526910c0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/fc8a6921de8b4783a6751994cb0aeef8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/ff3700dcd42349349cb5ac9de3c037b8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/81e926bed7084fd8a32828749c3ae683, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/af60a9b0c3914107bb59624fe3eb8ba8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/fcb073fe992c4b2583fbf13bebb7bbd2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/d8959ebb7eb94058a8ed15cf7e90ff7c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/26809c138efd4a73984ec03662d36f02, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/94e4e476076447018af2fbf81e3404e7, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/3711a8ce3c0546f0ab8e1a7e24c103cb, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/28e06b233d4c42d08a25d8617230015d] to archive 2024-11-18T07:53:55,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:53:55,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/495bd213ab494c38b73b71f01be6fd9f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/495bd213ab494c38b73b71f01be6fd9f 2024-11-18T07:53:55,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/b16dbc85b82545a0a7434b73e032394f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/b16dbc85b82545a0a7434b73e032394f 2024-11-18T07:53:55,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/8c73c67cedfd4b08938dcd9d6cd5d874 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/8c73c67cedfd4b08938dcd9d6cd5d874 2024-11-18T07:53:55,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/5bc955c704d14e08a6d5b86deb15f3c4 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/5bc955c704d14e08a6d5b86deb15f3c4 2024-11-18T07:53:55,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/7f59dddd10964800b9c6f4abd02cafa8 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/7f59dddd10964800b9c6f4abd02cafa8 2024-11-18T07:53:55,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/76439e38243c4f2d833c4ee4059fa3c3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/76439e38243c4f2d833c4ee4059fa3c3 2024-11-18T07:53:55,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/c9abc1b2a47c48bdafd54355985403fc to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/c9abc1b2a47c48bdafd54355985403fc 2024-11-18T07:53:55,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/678e5430fceb433693761ee35e213db5 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/678e5430fceb433693761ee35e213db5 2024-11-18T07:53:55,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/537af44619704fd39c7729f0e8cebfb8 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/537af44619704fd39c7729f0e8cebfb8 2024-11-18T07:53:55,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/f5c4fa0a637c4a138f4abb90a8e8a53d to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/f5c4fa0a637c4a138f4abb90a8e8a53d 2024-11-18T07:53:55,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/4b9134b8cab64419ba910a92e1c8e853 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/4b9134b8cab64419ba910a92e1c8e853 2024-11-18T07:53:55,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/53be7f525eac4e09b9fb096c62748851 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/53be7f525eac4e09b9fb096c62748851 2024-11-18T07:53:55,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/7c60a93d8e1f4edaafb48569526910c0 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/7c60a93d8e1f4edaafb48569526910c0 2024-11-18T07:53:55,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/fc8a6921de8b4783a6751994cb0aeef8 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/fc8a6921de8b4783a6751994cb0aeef8 2024-11-18T07:53:55,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/ff3700dcd42349349cb5ac9de3c037b8 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/ff3700dcd42349349cb5ac9de3c037b8 2024-11-18T07:53:55,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/81e926bed7084fd8a32828749c3ae683 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/81e926bed7084fd8a32828749c3ae683 2024-11-18T07:53:55,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/af60a9b0c3914107bb59624fe3eb8ba8 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/af60a9b0c3914107bb59624fe3eb8ba8 2024-11-18T07:53:55,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/fcb073fe992c4b2583fbf13bebb7bbd2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/fcb073fe992c4b2583fbf13bebb7bbd2 2024-11-18T07:53:55,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/d8959ebb7eb94058a8ed15cf7e90ff7c to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/d8959ebb7eb94058a8ed15cf7e90ff7c 2024-11-18T07:53:55,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/26809c138efd4a73984ec03662d36f02 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/26809c138efd4a73984ec03662d36f02 2024-11-18T07:53:55,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/94e4e476076447018af2fbf81e3404e7 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/94e4e476076447018af2fbf81e3404e7 2024-11-18T07:53:55,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/3711a8ce3c0546f0ab8e1a7e24c103cb to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/3711a8ce3c0546f0ab8e1a7e24c103cb 2024-11-18T07:53:55,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/28e06b233d4c42d08a25d8617230015d to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/28e06b233d4c42d08a25d8617230015d 2024-11-18T07:53:55,443 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/recovered.edits/400.seqid, newMaxSeqId=400, maxSeqId=4 2024-11-18T07:53:55,444 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6. 2024-11-18T07:53:55,444 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1676): Region close journal for 4a6df03ed24f69072d2b4a08bac0a9a6: Waiting for close lock at 1731916434235Running coprocessor pre-close hooks at 1731916434235Disabling compacts and flushes for region at 1731916434235Disabling writes for close at 1731916435344 (+1109 ms)Writing region close event to WAL at 1731916435438 (+94 ms)Running coprocessor post-close hooks at 1731916435444 (+6 ms)Closed at 1731916435444 2024-11-18T07:53:55,451 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(157): Closed 4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:55,451 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=4a6df03ed24f69072d2b4a08bac0a9a6, regionState=CLOSED 2024-11-18T07:53:55,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:53:55,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-18T07:53:55,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a6df03ed24f69072d2b4a08bac0a9a6, server=87fb3d252950,37987,1731916323128 in 1.3740 sec 2024-11-18T07:53:55,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-18T07:53:55,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a6df03ed24f69072d2b4a08bac0a9a6, UNASSIGN in 1.3820 sec 2024-11-18T07:53:55,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-18T07:53:55,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3890 sec 2024-11-18T07:53:55,463 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916435462"}]},"ts":"1731916435462"} 2024-11-18T07:53:55,466 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T07:53:55,466 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T07:53:55,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4130 sec 2024-11-18T07:53:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T07:53:56,219 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:56,220 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T07:53:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:56,222 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T07:53:56,225 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=109, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:56,231 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,234 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/recovered.edits] 2024-11-18T07:53:56,246 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/51eda9a2014948018f494ba4e75b23e0 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/51eda9a2014948018f494ba4e75b23e0 2024-11-18T07:53:56,249 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/a564e6baa9784042baf7ef8de9b83b32 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/A/a564e6baa9784042baf7ef8de9b83b32 2024-11-18T07:53:56,256 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/47cfd3b8bc2a4da2a7fc68e57f9a7470 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/47cfd3b8bc2a4da2a7fc68e57f9a7470 2024-11-18T07:53:56,259 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/90626b6f80c343a4911203db2559b8af to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/B/90626b6f80c343a4911203db2559b8af 2024-11-18T07:53:56,265 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/3f2a3748e1974827a38b8cf819d8bcd3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/3f2a3748e1974827a38b8cf819d8bcd3 2024-11-18T07:53:56,269 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/f7c12361430f471b9cec6d84fbf9d076 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/C/f7c12361430f471b9cec6d84fbf9d076 2024-11-18T07:53:56,275 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/recovered.edits/400.seqid to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6/recovered.edits/400.seqid 2024-11-18T07:53:56,276 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,276 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T07:53:56,277 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T07:53:56,278 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T07:53:56,288 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118060cf54a0c9742f4a46471a72bdc3847_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118060cf54a0c9742f4a46471a72bdc3847_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,289 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181344127826ca4920ba76ed4e480bd13f_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181344127826ca4920ba76ed4e480bd13f_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,291 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184999581e58304487ba241b9562e4fc37_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184999581e58304487ba241b9562e4fc37_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,293 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184deeb3e5b1974dac9593eea965528734_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184deeb3e5b1974dac9593eea965528734_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,295 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185c2ad833c98946ce9c12605c3b19d27d_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185c2ad833c98946ce9c12605c3b19d27d_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,297 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185ca38f9a215546b9b83558fba3347a7f_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185ca38f9a215546b9b83558fba3347a7f_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,298 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185fd0b00b627f489c8f3efc84332b8c85_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185fd0b00b627f489c8f3efc84332b8c85_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,300 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187b9fc140560440148ce261613c2576bc_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187b9fc140560440148ce261613c2576bc_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,304 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187f448c2354664b5b9b8687e86c427e4a_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187f448c2354664b5b9b8687e86c427e4a_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,306 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188408631c2b2c4c3989ea46b89f65281b_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188408631c2b2c4c3989ea46b89f65281b_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,308 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118925877c6e20349808478eeff0ba39136_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118925877c6e20349808478eeff0ba39136_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,311 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111898bd91a046e644cc9cf7e1122c5a12a9_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111898bd91a046e644cc9cf7e1122c5a12a9_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,314 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ac54a475216348aa998f0cb79113bd7c_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ac54a475216348aa998f0cb79113bd7c_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,316 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c0426c31a9af4f8099dd64df22553f17_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c0426c31a9af4f8099dd64df22553f17_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,320 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d2a41912e5164383bf92c352cc81798f_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d2a41912e5164383bf92c352cc81798f_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,322 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e1084db790ad4de3b33a3ebb7012881e_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e1084db790ad4de3b33a3ebb7012881e_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,325 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f0e6af07834a4adc8df12fc6dd4db09a_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f0e6af07834a4adc8df12fc6dd4db09a_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T07:53:56,329 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f220ad28177649ca90687ac53f7a4dca_4a6df03ed24f69072d2b4a08bac0a9a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f220ad28177649ca90687ac53f7a4dca_4a6df03ed24f69072d2b4a08bac0a9a6 2024-11-18T07:53:56,330 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T07:53:56,341 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=109, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:56,344 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T07:53:56,347 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T07:53:56,349 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=109, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:56,349 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T07:53:56,350 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731916436349"}]},"ts":"9223372036854775807"} 2024-11-18T07:53:56,352 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T07:53:56,352 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4a6df03ed24f69072d2b4a08bac0a9a6, NAME => 'TestAcidGuarantees,,1731916409484.4a6df03ed24f69072d2b4a08bac0a9a6.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T07:53:56,352 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T07:53:56,353 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731916436352"}]},"ts":"9223372036854775807"} 2024-11-18T07:53:56,355 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T07:53:56,356 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=109, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:56,359 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 137 msec 2024-11-18T07:53:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T07:53:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T07:53:56,538 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:56,553 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 250) - Thread LEAK? -, OpenFileDescriptor=461 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1105 (was 1058) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1846 (was 1903) 2024-11-18T07:53:56,564 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=1105, ProcessCount=11, AvailableMemoryMB=1846 2024-11-18T07:53:56,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T07:53:56,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T07:53:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T07:53:56,571 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T07:53:56,571 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:56,574 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 110 2024-11-18T07:53:56,576 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T07:53:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-18T07:53:56,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742238_1414 (size=1061) 2024-11-18T07:53:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-18T07:53:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-18T07:53:57,020 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3 2024-11-18T07:53:57,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742239_1415 (size=53) 2024-11-18T07:53:57,063 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:53:57,063 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing deeef93d343bf8d154ce5359480346cc, disabling compactions & flushes 2024-11-18T07:53:57,063 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:57,063 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:57,063 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. after waiting 0 ms 2024-11-18T07:53:57,063 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:57,063 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:57,063 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for deeef93d343bf8d154ce5359480346cc: Waiting for close lock at 1731916437063Disabling compacts and flushes for region at 1731916437063Disabling writes for close at 1731916437063Writing region close event to WAL at 1731916437063Closed at 1731916437063 2024-11-18T07:53:57,065 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T07:53:57,065 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731916437065"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731916437065"}]},"ts":"1731916437065"} 2024-11-18T07:53:57,067 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T07:53:57,068 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T07:53:57,069 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916437068"}]},"ts":"1731916437068"} 2024-11-18T07:53:57,071 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T07:53:57,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=deeef93d343bf8d154ce5359480346cc, ASSIGN}] 2024-11-18T07:53:57,073 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=deeef93d343bf8d154ce5359480346cc, ASSIGN 2024-11-18T07:53:57,074 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=deeef93d343bf8d154ce5359480346cc, ASSIGN; state=OFFLINE, location=87fb3d252950,37987,1731916323128; forceNewPlan=false, retain=false 2024-11-18T07:53:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-18T07:53:57,225 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=deeef93d343bf8d154ce5359480346cc, regionState=OPENING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:53:57,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=deeef93d343bf8d154ce5359480346cc, ASSIGN because future has completed 2024-11-18T07:53:57,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:53:57,392 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:57,393 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7752): Opening region: {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} 2024-11-18T07:53:57,393 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees deeef93d343bf8d154ce5359480346cc 2024-11-18T07:53:57,394 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:53:57,394 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7794): checking encryption for deeef93d343bf8d154ce5359480346cc 2024-11-18T07:53:57,394 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7797): checking classloading for deeef93d343bf8d154ce5359480346cc 2024-11-18T07:53:57,403 INFO [StoreOpener-deeef93d343bf8d154ce5359480346cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region deeef93d343bf8d154ce5359480346cc 2024-11-18T07:53:57,410 INFO [StoreOpener-deeef93d343bf8d154ce5359480346cc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:53:57,411 INFO [StoreOpener-deeef93d343bf8d154ce5359480346cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region deeef93d343bf8d154ce5359480346cc columnFamilyName A 2024-11-18T07:53:57,412 DEBUG [StoreOpener-deeef93d343bf8d154ce5359480346cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:57,413 INFO [StoreOpener-deeef93d343bf8d154ce5359480346cc-1 {}] regionserver.HStore(327): Store=deeef93d343bf8d154ce5359480346cc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:53:57,415 INFO [StoreOpener-deeef93d343bf8d154ce5359480346cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region deeef93d343bf8d154ce5359480346cc 2024-11-18T07:53:57,419 INFO [StoreOpener-deeef93d343bf8d154ce5359480346cc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:53:57,419 INFO [StoreOpener-deeef93d343bf8d154ce5359480346cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region deeef93d343bf8d154ce5359480346cc columnFamilyName B 2024-11-18T07:53:57,419 DEBUG [StoreOpener-deeef93d343bf8d154ce5359480346cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:57,421 INFO [StoreOpener-deeef93d343bf8d154ce5359480346cc-1 {}] regionserver.HStore(327): Store=deeef93d343bf8d154ce5359480346cc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:53:57,421 INFO [StoreOpener-deeef93d343bf8d154ce5359480346cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region deeef93d343bf8d154ce5359480346cc 2024-11-18T07:53:57,427 INFO [StoreOpener-deeef93d343bf8d154ce5359480346cc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:53:57,427 INFO [StoreOpener-deeef93d343bf8d154ce5359480346cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region deeef93d343bf8d154ce5359480346cc columnFamilyName C 2024-11-18T07:53:57,427 DEBUG [StoreOpener-deeef93d343bf8d154ce5359480346cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:53:57,428 INFO [StoreOpener-deeef93d343bf8d154ce5359480346cc-1 {}] regionserver.HStore(327): Store=deeef93d343bf8d154ce5359480346cc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:53:57,428 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:57,428 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1038): replaying wal for deeef93d343bf8d154ce5359480346cc 2024-11-18T07:53:57,429 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc 2024-11-18T07:53:57,430 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc 2024-11-18T07:53:57,431 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1048): stopping wal replay for deeef93d343bf8d154ce5359480346cc 2024-11-18T07:53:57,431 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1060): Cleaning up temporary data for deeef93d343bf8d154ce5359480346cc 2024-11-18T07:53:57,432 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T07:53:57,433 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1093): writing seq id for deeef93d343bf8d154ce5359480346cc 2024-11-18T07:53:57,437 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T07:53:57,438 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1114): Opened deeef93d343bf8d154ce5359480346cc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65145626, jitterRate=-0.029254525899887085}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T07:53:57,438 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1122): Running coprocessor post-open hooks for deeef93d343bf8d154ce5359480346cc 2024-11-18T07:53:57,439 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1006): Region open journal for deeef93d343bf8d154ce5359480346cc: Running coprocessor pre-open hook at 1731916437394Writing region info on filesystem at 1731916437394Initializing all the Stores at 1731916437399 (+5 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916437399Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916437402 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916437402Cleaning up temporary data from old regions at 1731916437431 (+29 ms)Running coprocessor post-open hooks at 1731916437438 (+7 ms)Region opened successfully at 1731916437439 (+1 ms) 2024-11-18T07:53:57,440 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., pid=112, masterSystemTime=1731916437382 2024-11-18T07:53:57,442 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:57,442 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:57,443 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=deeef93d343bf8d154ce5359480346cc, regionState=OPEN, openSeqNum=2, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:53:57,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:53:57,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-18T07:53:57,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; OpenRegionProcedure deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 in 221 msec 2024-11-18T07:53:57,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-18T07:53:57,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=deeef93d343bf8d154ce5359480346cc, ASSIGN in 379 msec 2024-11-18T07:53:57,455 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T07:53:57,455 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916437455"}]},"ts":"1731916437455"} 2024-11-18T07:53:57,457 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T07:53:57,458 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T07:53:57,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 891 msec 2024-11-18T07:53:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-18T07:53:57,719 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:57,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@269ecf23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,724 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:57,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:57,725 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:57,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:57,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:57,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3531e96f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:57,727 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:57,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37d3c6c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,728 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:57,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:57,736 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T07:53:57,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 87fb3d252950,42633,1731916322278 2024-11-18T07:53:57,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@52c55446 2024-11-18T07:53:57,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T07:53:57,739 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T07:53:57,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594674bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,743 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:57,743 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:57,744 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dcda11c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:57,747 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:57,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d5e338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@119ad987, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,749 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:57,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:57,750 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:57,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:57,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3490ad67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:57,752 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:57,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d055805, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c4ef34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:57,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:57,755 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:57,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:57,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:57,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f435357, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:57,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:57,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:57,757 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69410aff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fd1f545, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:57,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:57,760 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b6fa19a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:57,762 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:57,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cd2b07e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c19380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:57,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:57,766 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ed94d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:57,768 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e9fc92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1627c26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:57,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:57,771 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@575e2730, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:57,774 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29da26cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5993889b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,776 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:57,776 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:57,777 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:57,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:57,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:57,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:57,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f32ed9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,778 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:57,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:57,780 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32445f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cf53828, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:57,783 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:57,784 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:57,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:57,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:57,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47cc34fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:57,785 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:57,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab0f0f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf240ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,787 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:57,787 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:57,788 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d38b825, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:57,790 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:57,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58283d6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3292b9b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,792 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:53:57,792 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:53:57,793 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:53:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:53:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:53:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@201f8219, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:53:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:53:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:53:57,794 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:53:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3379290e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:53:57,796 DEBUG [Thread-1881 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:57,798 DEBUG [Thread-1888 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:57,799 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-18T07:53:57,800 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:57,801 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T07:53:57,803 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:57,804 DEBUG [Thread-1886 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:57,804 DEBUG [Thread-1890 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:57,804 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:57,804 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:53:57,807 DEBUG [Thread-1892 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:57,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:57,808 DEBUG [Thread-1890 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:57,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:57,808 DEBUG [Thread-1890 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:57,808 DEBUG [Thread-1875 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:57,808 DEBUG [Thread-1886 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:57,808 DEBUG [Thread-1886 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:57,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:57,810 DEBUG [Thread-1883 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:57,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:57,811 DEBUG [Thread-1879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:57,813 DEBUG [Thread-1877 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:57,813 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:57,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:57,815 DEBUG [Thread-1877 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:57,815 DEBUG [Thread-1877 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:57,816 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:57,816 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:57,824 DEBUG [Thread-1883 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:57,824 DEBUG [Thread-1883 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:57,827 DEBUG [Thread-1894 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:53:57,828 DEBUG [Thread-1894 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:53:57,829 DEBUG [Thread-1894 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:53:57,830 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:57,831 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:53:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2] 2024-11-18T07:53:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:53:57,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:53:57,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:53:57,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:57,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:53:57,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:57,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:53:57,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:57,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/a76beb34e7aa4b01a21bdec5f7827f84 is 50, key is test_row_0/A:col10/1731916437831/Put/seqid=0 2024-11-18T07:53:57,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T07:53:57,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742240_1416 (size=14341) 2024-11-18T07:53:57,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:57,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:57,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916497920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:57,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:57,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:57,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:57,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37826 deadline: 1731916497921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:57,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:57,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916497922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:57,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:57,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:57,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:57,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:57,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:57,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:57,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731916437906, max timestamp=1731916437939Num uniques 859; 2024-11-18T07:53:57,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:53:57,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731916437906, max timestamp=1731916437939Num uniques 859; 2024-11-18T07:53:57,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:53:57,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731916437906, max timestamp=1731916437939Num uniques 859; 2024-11-18T07:53:57,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T07:53:57,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T07:53:57,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:57,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:53:57,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:57,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:57,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:57,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916498058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916498077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916498084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916498086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37826 deadline: 1731916498092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T07:53:58,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:58,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:53:58,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:58,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T07:53:58,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916498188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916498211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916498211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916498211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37826 deadline: 1731916498219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T07:53:58,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:58,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:53:58,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:58,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/a76beb34e7aa4b01a21bdec5f7827f84 2024-11-18T07:53:58,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/0785f910bf614c85a98d8fcc97e0982d is 50, key is test_row_0/B:col10/1731916437831/Put/seqid=0 2024-11-18T07:53:58,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T07:53:58,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:58,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:53:58,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:58,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T07:53:58,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916498419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916498438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916498438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916498438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742241_1417 (size=12001) 2024-11-18T07:53:58,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37826 deadline: 1731916498439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/0785f910bf614c85a98d8fcc97e0982d 2024-11-18T07:53:58,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/94a3c5aa864c4511ae45434ce66aa45d is 50, key is test_row_0/C:col10/1731916437831/Put/seqid=0 2024-11-18T07:53:58,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742242_1418 (size=12001) 2024-11-18T07:53:58,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T07:53:58,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:58,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:53:58,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:58,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T07:53:58,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:58,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:53:58,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:58,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916498748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916498758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916498758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916498759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37826 deadline: 1731916498778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:58,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T07:53:58,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:58,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:53:58,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:58,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:53:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T07:53:58,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/94a3c5aa864c4511ae45434ce66aa45d 2024-11-18T07:53:58,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/a76beb34e7aa4b01a21bdec5f7827f84 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/a76beb34e7aa4b01a21bdec5f7827f84 2024-11-18T07:53:59,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/a76beb34e7aa4b01a21bdec5f7827f84, entries=200, sequenceid=12, filesize=14.0 K 2024-11-18T07:53:59,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/0785f910bf614c85a98d8fcc97e0982d as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/0785f910bf614c85a98d8fcc97e0982d 2024-11-18T07:53:59,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/0785f910bf614c85a98d8fcc97e0982d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-18T07:53:59,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/94a3c5aa864c4511ae45434ce66aa45d as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/94a3c5aa864c4511ae45434ce66aa45d 2024-11-18T07:53:59,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/94a3c5aa864c4511ae45434ce66aa45d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-18T07:53:59,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for deeef93d343bf8d154ce5359480346cc in 1212ms, sequenceid=12, compaction requested=false 2024-11-18T07:53:59,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:53:59,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T07:53:59,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:59,058 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-18T07:53:59,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:53:59,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:59,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:53:59,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:59,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:53:59,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:53:59,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/b93a17926f4841a98ed8e9cbcba8cc0c is 50, key is test_row_0/A:col10/1731916438077/Put/seqid=0 2024-11-18T07:53:59,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742243_1419 (size=12001) 2024-11-18T07:53:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:53:59,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:53:59,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916499294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916499296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37826 deadline: 1731916499298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916499299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916499317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916499429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916499429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916499429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916499442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,510 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/b93a17926f4841a98ed8e9cbcba8cc0c 2024-11-18T07:53:59,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/7acc72b874a7435898f4f81097456976 is 50, key is test_row_0/B:col10/1731916438077/Put/seqid=0 2024-11-18T07:53:59,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742244_1420 (size=12001) 2024-11-18T07:53:59,576 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/7acc72b874a7435898f4f81097456976 2024-11-18T07:53:59,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/5ab090d4ccf941fea30ecdc2f803d210 is 50, key is test_row_0/C:col10/1731916438077/Put/seqid=0 2024-11-18T07:53:59,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742245_1421 (size=12001) 2024-11-18T07:53:59,614 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/5ab090d4ccf941fea30ecdc2f803d210 2024-11-18T07:53:59,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/b93a17926f4841a98ed8e9cbcba8cc0c as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/b93a17926f4841a98ed8e9cbcba8cc0c 2024-11-18T07:53:59,623 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/b93a17926f4841a98ed8e9cbcba8cc0c, entries=150, sequenceid=48, filesize=11.7 K 2024-11-18T07:53:59,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/7acc72b874a7435898f4f81097456976 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/7acc72b874a7435898f4f81097456976 2024-11-18T07:53:59,629 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/7acc72b874a7435898f4f81097456976, entries=150, sequenceid=48, filesize=11.7 K 2024-11-18T07:53:59,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/5ab090d4ccf941fea30ecdc2f803d210 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/5ab090d4ccf941fea30ecdc2f803d210 2024-11-18T07:53:59,635 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/5ab090d4ccf941fea30ecdc2f803d210, entries=150, sequenceid=48, filesize=11.7 K 2024-11-18T07:53:59,636 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for deeef93d343bf8d154ce5359480346cc in 578ms, sequenceid=48, compaction requested=false 2024-11-18T07:53:59,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:53:59,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:53:59,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-18T07:53:59,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-18T07:53:59,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-18T07:53:59,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8340 sec 2024-11-18T07:53:59,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.8420 sec 2024-11-18T07:53:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:53:59,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T07:53:59,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:53:59,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:59,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:53:59,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:59,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:53:59,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:53:59,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/3083bd5874704d30bfb945b1e149b8d8 is 50, key is test_row_0/A:col10/1731916439662/Put/seqid=0 2024-11-18T07:53:59,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742246_1422 (size=14341) 2024-11-18T07:53:59,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/3083bd5874704d30bfb945b1e149b8d8 2024-11-18T07:53:59,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/fa3984c8824a454badc9c96dfe501913 is 50, key is test_row_0/B:col10/1731916439662/Put/seqid=0 2024-11-18T07:53:59,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916499749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742247_1423 (size=12001) 2024-11-18T07:53:59,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916499755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916499757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916499759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916499868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916499888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916499888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:53:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916499888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:53:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:53:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:53:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T07:53:59,949 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:53:59,954 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:53:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-18T07:53:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T07:53:59,959 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:53:59,960 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:53:59,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T07:54:00,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-18T07:54:00,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:00,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:00,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:00,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:00,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:00,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916500098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916500119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916500119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916500119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/fa3984c8824a454badc9c96dfe501913 2024-11-18T07:54:00,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-18T07:54:00,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:00,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:00,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:00,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:00,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:00,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/d7c7ac79719449f799ffd3a8566c253b is 50, key is test_row_0/C:col10/1731916439662/Put/seqid=0 2024-11-18T07:54:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T07:54:00,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742248_1424 (size=12001) 2024-11-18T07:54:00,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/d7c7ac79719449f799ffd3a8566c253b 2024-11-18T07:54:00,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37826 deadline: 1731916500329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/3083bd5874704d30bfb945b1e149b8d8 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/3083bd5874704d30bfb945b1e149b8d8 2024-11-18T07:54:00,354 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_1' in deeef93d343bf8d154ce5359480346cc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:54:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/3083bd5874704d30bfb945b1e149b8d8, entries=200, sequenceid=60, filesize=14.0 K 2024-11-18T07:54:00,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/fa3984c8824a454badc9c96dfe501913 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fa3984c8824a454badc9c96dfe501913 2024-11-18T07:54:00,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fa3984c8824a454badc9c96dfe501913, entries=150, sequenceid=60, filesize=11.7 K 2024-11-18T07:54:00,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/d7c7ac79719449f799ffd3a8566c253b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d7c7ac79719449f799ffd3a8566c253b 2024-11-18T07:54:00,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d7c7ac79719449f799ffd3a8566c253b, entries=150, sequenceid=60, filesize=11.7 K 2024-11-18T07:54:00,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for deeef93d343bf8d154ce5359480346cc in 725ms, sequenceid=60, compaction requested=true 2024-11-18T07:54:00,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:00,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:00,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:00,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:00,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:54:00,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:00,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T07:54:00,394 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:00,394 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:00,399 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:00,399 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/C is initiating minor compaction (all files) 2024-11-18T07:54:00,400 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/C in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:00,400 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/94a3c5aa864c4511ae45434ce66aa45d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/5ab090d4ccf941fea30ecdc2f803d210, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d7c7ac79719449f799ffd3a8566c253b] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=35.2 K 2024-11-18T07:54:00,400 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 94a3c5aa864c4511ae45434ce66aa45d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731916437822 2024-11-18T07:54:00,400 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:00,400 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/A is initiating minor compaction (all files) 2024-11-18T07:54:00,401 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/A in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:00,401 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/a76beb34e7aa4b01a21bdec5f7827f84, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/b93a17926f4841a98ed8e9cbcba8cc0c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/3083bd5874704d30bfb945b1e149b8d8] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=39.7 K 2024-11-18T07:54:00,403 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ab090d4ccf941fea30ecdc2f803d210, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731916438074 2024-11-18T07:54:00,403 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a76beb34e7aa4b01a21bdec5f7827f84, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731916437822 2024-11-18T07:54:00,403 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting d7c7ac79719449f799ffd3a8566c253b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916439271 2024-11-18T07:54:00,403 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b93a17926f4841a98ed8e9cbcba8cc0c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731916438074 2024-11-18T07:54:00,407 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3083bd5874704d30bfb945b1e149b8d8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916439271 2024-11-18T07:54:00,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-18T07:54:00,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:00,427 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T07:54:00,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:00,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:00,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:00,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:00,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:00,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:00,430 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#C#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:00,431 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/6dfbda4c067c47a4ae75941ad0543c8f is 50, key is test_row_0/C:col10/1731916439662/Put/seqid=0 2024-11-18T07:54:00,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:00,473 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#A#compaction#352 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:00,474 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/b7f3506d70c64cdbb5e1dd3df2916363 is 50, key is test_row_0/A:col10/1731916439662/Put/seqid=0 2024-11-18T07:54:00,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/823fa50cdf1b492898f6adcb525c6394 is 50, key is test_row_0/A:col10/1731916439753/Put/seqid=0 2024-11-18T07:54:00,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742249_1425 (size=12104) 2024-11-18T07:54:00,539 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/6dfbda4c067c47a4ae75941ad0543c8f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6dfbda4c067c47a4ae75941ad0543c8f 2024-11-18T07:54:00,556 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/C of deeef93d343bf8d154ce5359480346cc into 6dfbda4c067c47a4ae75941ad0543c8f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:00,556 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:00,556 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/C, priority=13, startTime=1731916440394; duration=0sec 2024-11-18T07:54:00,556 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:00,556 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:C 2024-11-18T07:54:00,556 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:00,557 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:00,557 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/B is initiating minor compaction (all files) 2024-11-18T07:54:00,557 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/B in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:00,557 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/0785f910bf614c85a98d8fcc97e0982d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/7acc72b874a7435898f4f81097456976, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fa3984c8824a454badc9c96dfe501913] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=35.2 K 2024-11-18T07:54:00,558 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0785f910bf614c85a98d8fcc97e0982d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731916437822 2024-11-18T07:54:00,558 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7acc72b874a7435898f4f81097456976, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731916438074 2024-11-18T07:54:00,559 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting fa3984c8824a454badc9c96dfe501913, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916439271 2024-11-18T07:54:00,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916500535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916500539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916500540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916500542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742250_1426 (size=12104) 2024-11-18T07:54:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T07:54:00,592 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/b7f3506d70c64cdbb5e1dd3df2916363 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/b7f3506d70c64cdbb5e1dd3df2916363 2024-11-18T07:54:00,607 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/A of deeef93d343bf8d154ce5359480346cc into b7f3506d70c64cdbb5e1dd3df2916363(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:00,608 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:00,608 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/A, priority=13, startTime=1731916440394; duration=0sec 2024-11-18T07:54:00,608 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:00,608 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:A 2024-11-18T07:54:00,620 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#B#compaction#354 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:00,620 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/26c6d06626cb4ed78ee1a6423b99ec5c is 50, key is test_row_0/B:col10/1731916439662/Put/seqid=0 2024-11-18T07:54:00,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742251_1427 (size=12001) 2024-11-18T07:54:00,631 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/823fa50cdf1b492898f6adcb525c6394 2024-11-18T07:54:00,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742252_1428 (size=12104) 2024-11-18T07:54:00,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/b7698079945143089db1adb30e1dc8a1 is 50, key is test_row_0/B:col10/1731916439753/Put/seqid=0 2024-11-18T07:54:00,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916500688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916500688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916500688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916500688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742253_1429 (size=12001) 2024-11-18T07:54:00,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916500908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916500919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916500919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:00,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916500919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:00,998 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T07:54:01,090 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/26c6d06626cb4ed78ee1a6423b99ec5c as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/26c6d06626cb4ed78ee1a6423b99ec5c 2024-11-18T07:54:01,095 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/B of deeef93d343bf8d154ce5359480346cc into 26c6d06626cb4ed78ee1a6423b99ec5c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:01,095 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:01,095 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/B, priority=13, startTime=1731916440394; duration=0sec 2024-11-18T07:54:01,095 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:01,095 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T07:54:01,140 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/b7698079945143089db1adb30e1dc8a1 2024-11-18T07:54:01,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/40b7a9d9232e4d77a6dad16fbe675850 is 50, key is test_row_0/C:col10/1731916439753/Put/seqid=0 2024-11-18T07:54:01,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742254_1430 (size=12001) 2024-11-18T07:54:01,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916501218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:01,236 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T07:54:01,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916501238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:01,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916501238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916501238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:01,569 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/40b7a9d9232e4d77a6dad16fbe675850 2024-11-18T07:54:01,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/823fa50cdf1b492898f6adcb525c6394 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/823fa50cdf1b492898f6adcb525c6394 2024-11-18T07:54:01,576 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/823fa50cdf1b492898f6adcb525c6394, entries=150, sequenceid=80, filesize=11.7 K 2024-11-18T07:54:01,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/b7698079945143089db1adb30e1dc8a1 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/b7698079945143089db1adb30e1dc8a1 2024-11-18T07:54:01,583 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/b7698079945143089db1adb30e1dc8a1, entries=150, sequenceid=80, filesize=11.7 K 2024-11-18T07:54:01,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/40b7a9d9232e4d77a6dad16fbe675850 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/40b7a9d9232e4d77a6dad16fbe675850 2024-11-18T07:54:01,596 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/40b7a9d9232e4d77a6dad16fbe675850, entries=150, sequenceid=80, filesize=11.7 K 2024-11-18T07:54:01,597 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for deeef93d343bf8d154ce5359480346cc in 1170ms, sequenceid=80, compaction requested=false 2024-11-18T07:54:01,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:01,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:01,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-18T07:54:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-18T07:54:01,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-18T07:54:01,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6410 sec 2024-11-18T07:54:01,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.6490 sec 2024-11-18T07:54:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:01,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T07:54:01,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:01,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:01,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:01,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:01,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:01,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:01,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/e7be41e073614bf89bac19508fce1681 is 50, key is test_row_0/A:col10/1731916441741/Put/seqid=0 2024-11-18T07:54:01,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742255_1431 (size=14341) 2024-11-18T07:54:01,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916501808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:01,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916501808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:01,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:01,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:01,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916501809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:01,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916501811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:01,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916501928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916501928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916501928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:01,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916501928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T07:54:02,118 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:02,119 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:54:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-18T07:54:02,121 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:54:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T07:54:02,122 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:54:02,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:02,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916502138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:02,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916502139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916502139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:02,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916502139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:02,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/e7be41e073614bf89bac19508fce1681 2024-11-18T07:54:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:02,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/282767d05b444291ac1488a0bddecdfa is 50, key is test_row_0/B:col10/1731916441741/Put/seqid=0 2024-11-18T07:54:02,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742256_1432 (size=12001) 2024-11-18T07:54:02,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/282767d05b444291ac1488a0bddecdfa 2024-11-18T07:54:02,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/f65ca6283e744576b99c52cf2100071b is 50, key is test_row_0/C:col10/1731916441741/Put/seqid=0 2024-11-18T07:54:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T07:54:02,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742257_1433 (size=12001) 2024-11-18T07:54:02,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-18T07:54:02,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:02,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:02,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:02,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:02,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:02,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37826 deadline: 1731916502368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:02,380 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_1' in deeef93d343bf8d154ce5359480346cc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4288 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:54:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:02,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-18T07:54:02,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:02,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:02,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:02,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:02,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:02,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:02,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T07:54:02,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:02,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916502459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:02,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:02,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916502459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:02,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916502459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:02,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916502459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:02,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-18T07:54:02,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:02,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:02,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:02,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:02,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:02,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/f65ca6283e744576b99c52cf2100071b 2024-11-18T07:54:02,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/e7be41e073614bf89bac19508fce1681 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/e7be41e073614bf89bac19508fce1681 2024-11-18T07:54:02,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/e7be41e073614bf89bac19508fce1681, entries=200, sequenceid=96, filesize=14.0 K 2024-11-18T07:54:02,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/282767d05b444291ac1488a0bddecdfa as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/282767d05b444291ac1488a0bddecdfa 2024-11-18T07:54:02,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/282767d05b444291ac1488a0bddecdfa, entries=150, sequenceid=96, filesize=11.7 K 2024-11-18T07:54:02,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/f65ca6283e744576b99c52cf2100071b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/f65ca6283e744576b99c52cf2100071b 2024-11-18T07:54:02,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/f65ca6283e744576b99c52cf2100071b, entries=150, sequenceid=96, filesize=11.7 K 2024-11-18T07:54:02,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for deeef93d343bf8d154ce5359480346cc in 950ms, sequenceid=96, compaction requested=true 2024-11-18T07:54:02,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:02,692 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:02,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:02,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:02,692 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:02,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:02,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:02,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:02,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:02,694 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:02,694 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/A is initiating minor compaction (all files) 2024-11-18T07:54:02,694 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/A in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:02,694 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/b7f3506d70c64cdbb5e1dd3df2916363, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/823fa50cdf1b492898f6adcb525c6394, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/e7be41e073614bf89bac19508fce1681] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=37.5 K 2024-11-18T07:54:02,694 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:02,694 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/B is initiating minor compaction (all files) 2024-11-18T07:54:02,694 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/B in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:02,695 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/26c6d06626cb4ed78ee1a6423b99ec5c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/b7698079945143089db1adb30e1dc8a1, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/282767d05b444291ac1488a0bddecdfa] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=35.3 K 2024-11-18T07:54:02,695 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 26c6d06626cb4ed78ee1a6423b99ec5c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916439271 2024-11-18T07:54:02,696 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7f3506d70c64cdbb5e1dd3df2916363, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916439271 2024-11-18T07:54:02,696 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting b7698079945143089db1adb30e1dc8a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731916439753 2024-11-18T07:54:02,696 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 823fa50cdf1b492898f6adcb525c6394, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731916439753 2024-11-18T07:54:02,696 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 282767d05b444291ac1488a0bddecdfa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731916440510 2024-11-18T07:54:02,697 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7be41e073614bf89bac19508fce1681, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731916440510 2024-11-18T07:54:02,729 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#A#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:02,731 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/da1ff4fb7d4943e4b6ca713301fe5308 is 50, key is test_row_0/A:col10/1731916441741/Put/seqid=0 2024-11-18T07:54:02,737 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#B#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:02,737 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/0a38a2b8e9ed4f0cacacdc7db2708fbf is 50, key is test_row_0/B:col10/1731916441741/Put/seqid=0 2024-11-18T07:54:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T07:54:02,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-18T07:54:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:02,750 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T07:54:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:02,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742258_1434 (size=12207) 2024-11-18T07:54:02,793 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/da1ff4fb7d4943e4b6ca713301fe5308 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/da1ff4fb7d4943e4b6ca713301fe5308 2024-11-18T07:54:02,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/330bfa8f70de414898365df34a04f78a is 50, key is test_row_0/A:col10/1731916441786/Put/seqid=0 2024-11-18T07:54:02,798 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/A of deeef93d343bf8d154ce5359480346cc into da1ff4fb7d4943e4b6ca713301fe5308(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:02,798 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:02,798 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/A, priority=13, startTime=1731916442692; duration=0sec 2024-11-18T07:54:02,798 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:02,798 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:A 2024-11-18T07:54:02,798 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:02,800 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:02,800 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/C is initiating minor compaction (all files) 2024-11-18T07:54:02,800 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/C in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:02,800 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6dfbda4c067c47a4ae75941ad0543c8f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/40b7a9d9232e4d77a6dad16fbe675850, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/f65ca6283e744576b99c52cf2100071b] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=35.3 K 2024-11-18T07:54:02,801 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dfbda4c067c47a4ae75941ad0543c8f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731916439271 2024-11-18T07:54:02,801 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40b7a9d9232e4d77a6dad16fbe675850, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731916439753 2024-11-18T07:54:02,801 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f65ca6283e744576b99c52cf2100071b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731916440510 2024-11-18T07:54:02,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742259_1435 (size=12207) 2024-11-18T07:54:02,828 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#C#compaction#363 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:02,828 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/9d13692a2fd1430ba1df504a6f18e131 is 50, key is test_row_0/C:col10/1731916441741/Put/seqid=0 2024-11-18T07:54:02,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742260_1436 (size=12001) 2024-11-18T07:54:02,855 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/330bfa8f70de414898365df34a04f78a 2024-11-18T07:54:02,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742261_1437 (size=12207) 2024-11-18T07:54:02,875 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/9d13692a2fd1430ba1df504a6f18e131 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/9d13692a2fd1430ba1df504a6f18e131 2024-11-18T07:54:02,880 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/C of deeef93d343bf8d154ce5359480346cc into 9d13692a2fd1430ba1df504a6f18e131(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:02,880 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:02,880 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/C, priority=13, startTime=1731916442692; duration=0sec 2024-11-18T07:54:02,880 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:02,880 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:C 2024-11-18T07:54:02,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/e765e62a3eed4710be15441b6942ecdf is 50, key is test_row_0/B:col10/1731916441786/Put/seqid=0 2024-11-18T07:54:02,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742262_1438 (size=12001) 2024-11-18T07:54:02,929 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/e765e62a3eed4710be15441b6942ecdf 2024-11-18T07:54:02,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/1dfc34f887634686b045a1a5957b56fd is 50, key is test_row_0/C:col10/1731916441786/Put/seqid=0 2024-11-18T07:54:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:02,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:02,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742263_1439 (size=12001) 2024-11-18T07:54:03,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916503001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916503007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916503010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916503012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916503118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916503128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916503138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916503139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,233 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/0a38a2b8e9ed4f0cacacdc7db2708fbf as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/0a38a2b8e9ed4f0cacacdc7db2708fbf 2024-11-18T07:54:03,241 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/B of deeef93d343bf8d154ce5359480346cc into 0a38a2b8e9ed4f0cacacdc7db2708fbf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:03,241 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:03,241 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/B, priority=13, startTime=1731916442692; duration=0sec 2024-11-18T07:54:03,241 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:03,241 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T07:54:03,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916503328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916503348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916503348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916503358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,386 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/1dfc34f887634686b045a1a5957b56fd 2024-11-18T07:54:03,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/330bfa8f70de414898365df34a04f78a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/330bfa8f70de414898365df34a04f78a 2024-11-18T07:54:03,395 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/330bfa8f70de414898365df34a04f78a, entries=150, sequenceid=115, filesize=11.7 K 2024-11-18T07:54:03,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/e765e62a3eed4710be15441b6942ecdf as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/e765e62a3eed4710be15441b6942ecdf 2024-11-18T07:54:03,403 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/e765e62a3eed4710be15441b6942ecdf, entries=150, sequenceid=115, filesize=11.7 K 2024-11-18T07:54:03,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/1dfc34f887634686b045a1a5957b56fd as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/1dfc34f887634686b045a1a5957b56fd 2024-11-18T07:54:03,409 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/1dfc34f887634686b045a1a5957b56fd, entries=150, sequenceid=115, filesize=11.7 K 2024-11-18T07:54:03,414 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for deeef93d343bf8d154ce5359480346cc in 663ms, sequenceid=115, compaction requested=false 2024-11-18T07:54:03,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:03,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:03,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-18T07:54:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-18T07:54:03,417 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-18T07:54:03,417 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2920 sec 2024-11-18T07:54:03,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.2980 sec 2024-11-18T07:54:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:03,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T07:54:03,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:03,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:03,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:03,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:03,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:03,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:03,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/2f14fcd86ab94e0cab9e07bf77727300 is 50, key is test_row_0/A:col10/1731916443008/Put/seqid=0 2024-11-18T07:54:03,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742264_1440 (size=14441) 2024-11-18T07:54:03,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916503691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916503698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916503699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916503700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916503808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916503818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916503819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916503819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:04,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916504017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:04,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916504028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:04,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:04,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916504038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:04,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916504039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:04,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:04,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/2f14fcd86ab94e0cab9e07bf77727300 2024-11-18T07:54:04,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/3f8f684cd6804a46a41019c8a6a96977 is 50, key is test_row_0/B:col10/1731916443008/Put/seqid=0 2024-11-18T07:54:04,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742265_1441 (size=12051) 2024-11-18T07:54:04,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/3f8f684cd6804a46a41019c8a6a96977 2024-11-18T07:54:04,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/76f4d5195d2440c182368ad02de2e66d is 50, key is test_row_0/C:col10/1731916443008/Put/seqid=0 2024-11-18T07:54:04,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742266_1442 (size=12051) 2024-11-18T07:54:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T07:54:04,269 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:04,269 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:54:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-18T07:54:04,271 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:54:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T07:54:04,272 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:54:04,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:04,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916504328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:04,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916504348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:04,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916504348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:04,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:04,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916504358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:04,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T07:54:04,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T07:54:04,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:04,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:04,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:04,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:04,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:04,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/76f4d5195d2440c182368ad02de2e66d 2024-11-18T07:54:04,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/2f14fcd86ab94e0cab9e07bf77727300 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/2f14fcd86ab94e0cab9e07bf77727300 2024-11-18T07:54:04,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/2f14fcd86ab94e0cab9e07bf77727300, entries=200, sequenceid=132, filesize=14.1 K 2024-11-18T07:54:04,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/3f8f684cd6804a46a41019c8a6a96977 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/3f8f684cd6804a46a41019c8a6a96977 2024-11-18T07:54:04,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/3f8f684cd6804a46a41019c8a6a96977, entries=150, sequenceid=132, filesize=11.8 K 2024-11-18T07:54:04,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/76f4d5195d2440c182368ad02de2e66d as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/76f4d5195d2440c182368ad02de2e66d 2024-11-18T07:54:04,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/76f4d5195d2440c182368ad02de2e66d, entries=150, sequenceid=132, filesize=11.8 K 2024-11-18T07:54:04,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for deeef93d343bf8d154ce5359480346cc in 878ms, sequenceid=132, compaction requested=true 2024-11-18T07:54:04,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:04,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:04,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:04,530 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:04,530 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:04,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:04,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:04,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:04,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:04,534 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:04,534 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/B is initiating minor compaction (all files) 2024-11-18T07:54:04,534 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/B in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:04,535 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/0a38a2b8e9ed4f0cacacdc7db2708fbf, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/e765e62a3eed4710be15441b6942ecdf, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/3f8f684cd6804a46a41019c8a6a96977] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=35.4 K 2024-11-18T07:54:04,535 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:04,535 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/A is initiating minor compaction (all files) 2024-11-18T07:54:04,535 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/A in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:04,535 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/da1ff4fb7d4943e4b6ca713301fe5308, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/330bfa8f70de414898365df34a04f78a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/2f14fcd86ab94e0cab9e07bf77727300] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=37.7 K 2024-11-18T07:54:04,535 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting da1ff4fb7d4943e4b6ca713301fe5308, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731916440510 2024-11-18T07:54:04,536 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a38a2b8e9ed4f0cacacdc7db2708fbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731916440510 2024-11-18T07:54:04,536 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 330bfa8f70de414898365df34a04f78a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731916441775 2024-11-18T07:54:04,536 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting e765e62a3eed4710be15441b6942ecdf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731916441775 2024-11-18T07:54:04,536 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f14fcd86ab94e0cab9e07bf77727300, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731916443005 2024-11-18T07:54:04,536 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f8f684cd6804a46a41019c8a6a96977, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731916443005 2024-11-18T07:54:04,559 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:04,559 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/fa4a4463b78e46b8a72448ff56122d74 is 50, key is test_row_0/B:col10/1731916443008/Put/seqid=0 2024-11-18T07:54:04,559 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#A#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:04,560 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/1b3762441d1e45c3979c666c9574d227 is 50, key is test_row_0/A:col10/1731916443008/Put/seqid=0 2024-11-18T07:54:04,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T07:54:04,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:04,579 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T07:54:04,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:04,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:04,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:04,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:04,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:04,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T07:54:04,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742267_1443 (size=12359) 2024-11-18T07:54:04,614 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/fa4a4463b78e46b8a72448ff56122d74 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fa4a4463b78e46b8a72448ff56122d74 2024-11-18T07:54:04,620 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/B of deeef93d343bf8d154ce5359480346cc into fa4a4463b78e46b8a72448ff56122d74(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:04,620 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:04,620 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/B, priority=13, startTime=1731916444530; duration=0sec 2024-11-18T07:54:04,620 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:04,620 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:04,620 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:04,624 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:04,625 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/C is initiating minor compaction (all files) 2024-11-18T07:54:04,625 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/C in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:04,625 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/9d13692a2fd1430ba1df504a6f18e131, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/1dfc34f887634686b045a1a5957b56fd, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/76f4d5195d2440c182368ad02de2e66d] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=35.4 K 2024-11-18T07:54:04,625 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d13692a2fd1430ba1df504a6f18e131, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731916440510 2024-11-18T07:54:04,626 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dfc34f887634686b045a1a5957b56fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731916441775 2024-11-18T07:54:04,626 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 76f4d5195d2440c182368ad02de2e66d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731916443005 2024-11-18T07:54:04,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742268_1444 (size=12359) 2024-11-18T07:54:04,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/11fd46ef45224fa8b925b8dbbcbea65a is 50, key is test_row_0/A:col10/1731916443699/Put/seqid=0 2024-11-18T07:54:04,650 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/1b3762441d1e45c3979c666c9574d227 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1b3762441d1e45c3979c666c9574d227 2024-11-18T07:54:04,657 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/A of deeef93d343bf8d154ce5359480346cc into 1b3762441d1e45c3979c666c9574d227(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:04,657 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:04,657 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/A, priority=13, startTime=1731916444530; duration=0sec 2024-11-18T07:54:04,657 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:04,657 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:A 2024-11-18T07:54:04,660 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#C#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:04,660 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/e974f635388d4402a3a6dd9fabfacfc9 is 50, key is test_row_0/C:col10/1731916443008/Put/seqid=0 2024-11-18T07:54:04,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742269_1445 (size=12151) 2024-11-18T07:54:04,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742270_1446 (size=12359) 2024-11-18T07:54:04,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:04,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T07:54:04,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916504892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:04,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:04,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916504894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:04,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:04,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916504898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:04,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:04,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916504906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:04,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:04,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:05,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916505007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:05,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:05,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916505018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:05,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916505018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:05,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:05,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916505028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:05,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:05,081 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/11fd46ef45224fa8b925b8dbbcbea65a 2024-11-18T07:54:05,085 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/e974f635388d4402a3a6dd9fabfacfc9 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/e974f635388d4402a3a6dd9fabfacfc9 2024-11-18T07:54:05,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/4347c206ad944cf0b7ae4414ed6c04ea is 50, key is test_row_0/B:col10/1731916443699/Put/seqid=0 2024-11-18T07:54:05,106 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/C of deeef93d343bf8d154ce5359480346cc into e974f635388d4402a3a6dd9fabfacfc9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:05,106 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:05,106 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/C, priority=13, startTime=1731916444531; duration=0sec 2024-11-18T07:54:05,106 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:05,106 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:C 2024-11-18T07:54:05,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742271_1447 (size=12151) 2024-11-18T07:54:05,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916505228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:05,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916505238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:05,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:05,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916505238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:05,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:05,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916505248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T07:54:05,521 WARN [regionserver/87fb3d252950:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-18T07:54:05,522 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/a76beb34e7aa4b01a21bdec5f7827f84, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/b93a17926f4841a98ed8e9cbcba8cc0c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/3083bd5874704d30bfb945b1e149b8d8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/b7f3506d70c64cdbb5e1dd3df2916363, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/823fa50cdf1b492898f6adcb525c6394, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/e7be41e073614bf89bac19508fce1681, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/da1ff4fb7d4943e4b6ca713301fe5308, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/330bfa8f70de414898365df34a04f78a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/2f14fcd86ab94e0cab9e07bf77727300] to archive 2024-11-18T07:54:05,523 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:54:05,527 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/4347c206ad944cf0b7ae4414ed6c04ea 2024-11-18T07:54:05,533 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/a76beb34e7aa4b01a21bdec5f7827f84 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/a76beb34e7aa4b01a21bdec5f7827f84 2024-11-18T07:54:05,535 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/b93a17926f4841a98ed8e9cbcba8cc0c to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/b93a17926f4841a98ed8e9cbcba8cc0c 2024-11-18T07:54:05,536 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/3083bd5874704d30bfb945b1e149b8d8 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/3083bd5874704d30bfb945b1e149b8d8 2024-11-18T07:54:05,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/82cd47cd819a48a2891e1f556337c824 is 50, key is test_row_0/C:col10/1731916443699/Put/seqid=0 2024-11-18T07:54:05,537 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/b7f3506d70c64cdbb5e1dd3df2916363 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/b7f3506d70c64cdbb5e1dd3df2916363 2024-11-18T07:54:05,541 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/823fa50cdf1b492898f6adcb525c6394 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/823fa50cdf1b492898f6adcb525c6394 2024-11-18T07:54:05,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/e7be41e073614bf89bac19508fce1681 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/e7be41e073614bf89bac19508fce1681 2024-11-18T07:54:05,543 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/da1ff4fb7d4943e4b6ca713301fe5308 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/da1ff4fb7d4943e4b6ca713301fe5308 2024-11-18T07:54:05,545 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/330bfa8f70de414898365df34a04f78a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/330bfa8f70de414898365df34a04f78a 2024-11-18T07:54:05,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916505539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:05,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:05,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/2f14fcd86ab94e0cab9e07bf77727300 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/2f14fcd86ab94e0cab9e07bf77727300 2024-11-18T07:54:05,554 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/0785f910bf614c85a98d8fcc97e0982d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/7acc72b874a7435898f4f81097456976, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/26c6d06626cb4ed78ee1a6423b99ec5c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fa3984c8824a454badc9c96dfe501913, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/b7698079945143089db1adb30e1dc8a1, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/0a38a2b8e9ed4f0cacacdc7db2708fbf, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/282767d05b444291ac1488a0bddecdfa, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/e765e62a3eed4710be15441b6942ecdf, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/3f8f684cd6804a46a41019c8a6a96977] to archive 2024-11-18T07:54:05,555 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:54:05,557 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/0785f910bf614c85a98d8fcc97e0982d to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/0785f910bf614c85a98d8fcc97e0982d 2024-11-18T07:54:05,559 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/7acc72b874a7435898f4f81097456976 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/7acc72b874a7435898f4f81097456976 2024-11-18T07:54:05,562 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/26c6d06626cb4ed78ee1a6423b99ec5c to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/26c6d06626cb4ed78ee1a6423b99ec5c 2024-11-18T07:54:05,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916505558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:05,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916505558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:05,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916505558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:05,565 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fa3984c8824a454badc9c96dfe501913 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fa3984c8824a454badc9c96dfe501913 2024-11-18T07:54:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:05,567 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/b7698079945143089db1adb30e1dc8a1 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/b7698079945143089db1adb30e1dc8a1 2024-11-18T07:54:05,569 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/0a38a2b8e9ed4f0cacacdc7db2708fbf to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/0a38a2b8e9ed4f0cacacdc7db2708fbf 2024-11-18T07:54:05,570 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/282767d05b444291ac1488a0bddecdfa to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/282767d05b444291ac1488a0bddecdfa 2024-11-18T07:54:05,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/e765e62a3eed4710be15441b6942ecdf to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/e765e62a3eed4710be15441b6942ecdf 2024-11-18T07:54:05,573 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/3f8f684cd6804a46a41019c8a6a96977 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/3f8f684cd6804a46a41019c8a6a96977 2024-11-18T07:54:05,579 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/94a3c5aa864c4511ae45434ce66aa45d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/5ab090d4ccf941fea30ecdc2f803d210, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6dfbda4c067c47a4ae75941ad0543c8f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d7c7ac79719449f799ffd3a8566c253b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/40b7a9d9232e4d77a6dad16fbe675850, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/9d13692a2fd1430ba1df504a6f18e131, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/f65ca6283e744576b99c52cf2100071b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/1dfc34f887634686b045a1a5957b56fd, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/76f4d5195d2440c182368ad02de2e66d] to archive 2024-11-18T07:54:05,579 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:54:05,581 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/94a3c5aa864c4511ae45434ce66aa45d to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/94a3c5aa864c4511ae45434ce66aa45d 2024-11-18T07:54:05,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742272_1448 (size=12151) 2024-11-18T07:54:05,583 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/5ab090d4ccf941fea30ecdc2f803d210 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/5ab090d4ccf941fea30ecdc2f803d210 2024-11-18T07:54:05,585 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6dfbda4c067c47a4ae75941ad0543c8f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6dfbda4c067c47a4ae75941ad0543c8f 2024-11-18T07:54:05,589 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d7c7ac79719449f799ffd3a8566c253b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d7c7ac79719449f799ffd3a8566c253b 2024-11-18T07:54:05,591 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/40b7a9d9232e4d77a6dad16fbe675850 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/40b7a9d9232e4d77a6dad16fbe675850 2024-11-18T07:54:05,592 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/9d13692a2fd1430ba1df504a6f18e131 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/9d13692a2fd1430ba1df504a6f18e131 2024-11-18T07:54:05,593 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/f65ca6283e744576b99c52cf2100071b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/f65ca6283e744576b99c52cf2100071b 2024-11-18T07:54:05,595 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/1dfc34f887634686b045a1a5957b56fd to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/1dfc34f887634686b045a1a5957b56fd 2024-11-18T07:54:05,596 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb3d252950:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/76f4d5195d2440c182368ad02de2e66d to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/76f4d5195d2440c182368ad02de2e66d 2024-11-18T07:54:05,982 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/82cd47cd819a48a2891e1f556337c824 2024-11-18T07:54:05,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/11fd46ef45224fa8b925b8dbbcbea65a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/11fd46ef45224fa8b925b8dbbcbea65a 2024-11-18T07:54:05,992 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/11fd46ef45224fa8b925b8dbbcbea65a, entries=150, sequenceid=150, filesize=11.9 K 2024-11-18T07:54:05,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/4347c206ad944cf0b7ae4414ed6c04ea as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/4347c206ad944cf0b7ae4414ed6c04ea 2024-11-18T07:54:05,996 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/4347c206ad944cf0b7ae4414ed6c04ea, entries=150, sequenceid=150, filesize=11.9 K 2024-11-18T07:54:05,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/82cd47cd819a48a2891e1f556337c824 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/82cd47cd819a48a2891e1f556337c824 2024-11-18T07:54:06,000 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/82cd47cd819a48a2891e1f556337c824, entries=150, sequenceid=150, filesize=11.9 K 2024-11-18T07:54:06,001 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for deeef93d343bf8d154ce5359480346cc in 1422ms, sequenceid=150, compaction requested=false 2024-11-18T07:54:06,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:06,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:06,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-18T07:54:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-18T07:54:06,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-18T07:54:06,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7300 sec 2024-11-18T07:54:06,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.7350 sec 2024-11-18T07:54:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:06,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T07:54:06,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:06,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:06,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:06,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:06,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:06,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:06,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/1480d781e694431981339386192cf5c9 is 50, key is test_row_0/A:col10/1731916444895/Put/seqid=0 2024-11-18T07:54:06,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742273_1449 (size=14541) 2024-11-18T07:54:06,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/1480d781e694431981339386192cf5c9 2024-11-18T07:54:06,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/530516bcd0524a319615739e1aa0c41f is 50, key is test_row_0/B:col10/1731916444895/Put/seqid=0 2024-11-18T07:54:06,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742274_1450 (size=12151) 2024-11-18T07:54:06,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/530516bcd0524a319615739e1aa0c41f 2024-11-18T07:54:06,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/261d30c401564939b23fa7b7f72197fa is 50, key is test_row_0/C:col10/1731916444895/Put/seqid=0 2024-11-18T07:54:06,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742275_1451 (size=12151) 2024-11-18T07:54:06,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916506096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916506104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916506109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916506112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916506218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916506219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916506228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916506238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T07:54:06,418 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:06,418 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:54:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-18T07:54:06,420 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:54:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T07:54:06,421 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:54:06,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:06,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37826 deadline: 1731916506428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916506428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916506428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,430 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_1' in deeef93d343bf8d154ce5359480346cc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8338 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:54:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916506438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916506458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/261d30c401564939b23fa7b7f72197fa 2024-11-18T07:54:06,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/1480d781e694431981339386192cf5c9 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1480d781e694431981339386192cf5c9 2024-11-18T07:54:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T07:54:06,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1480d781e694431981339386192cf5c9, entries=200, sequenceid=168, filesize=14.2 K 2024-11-18T07:54:06,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/530516bcd0524a319615739e1aa0c41f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/530516bcd0524a319615739e1aa0c41f 2024-11-18T07:54:06,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/530516bcd0524a319615739e1aa0c41f, entries=150, sequenceid=168, filesize=11.9 K 2024-11-18T07:54:06,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/261d30c401564939b23fa7b7f72197fa as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/261d30c401564939b23fa7b7f72197fa 2024-11-18T07:54:06,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/261d30c401564939b23fa7b7f72197fa, entries=150, sequenceid=168, filesize=11.9 K 2024-11-18T07:54:06,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for deeef93d343bf8d154ce5359480346cc in 500ms, sequenceid=168, compaction requested=true 2024-11-18T07:54:06,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:06,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:06,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:06,555 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:06,555 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:06,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:06,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:06,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:06,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:06,557 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:06,557 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/A is initiating minor compaction (all files) 2024-11-18T07:54:06,557 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/A in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:06,557 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1b3762441d1e45c3979c666c9574d227, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/11fd46ef45224fa8b925b8dbbcbea65a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1480d781e694431981339386192cf5c9] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=38.1 K 2024-11-18T07:54:06,557 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:06,557 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/B is initiating minor compaction (all files) 2024-11-18T07:54:06,557 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/B in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:06,557 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fa4a4463b78e46b8a72448ff56122d74, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/4347c206ad944cf0b7ae4414ed6c04ea, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/530516bcd0524a319615739e1aa0c41f] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=35.8 K 2024-11-18T07:54:06,558 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting fa4a4463b78e46b8a72448ff56122d74, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731916443005 2024-11-18T07:54:06,558 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b3762441d1e45c3979c666c9574d227, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731916443005 2024-11-18T07:54:06,558 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4347c206ad944cf0b7ae4414ed6c04ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731916443696 2024-11-18T07:54:06,559 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11fd46ef45224fa8b925b8dbbcbea65a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731916443696 2024-11-18T07:54:06,559 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 530516bcd0524a319615739e1aa0c41f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731916444890 2024-11-18T07:54:06,559 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1480d781e694431981339386192cf5c9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731916444890 2024-11-18T07:54:06,567 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#A#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:06,567 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/ba7522c7aafb42739561bd2e62c46db3 is 50, key is test_row_0/A:col10/1731916444895/Put/seqid=0 2024-11-18T07:54:06,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T07:54:06,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:06,574 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T07:54:06,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:06,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:06,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:06,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:06,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:06,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:06,581 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#B#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:06,581 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/2c6038cbcdc84db3ad9522613f7f764f is 50, key is test_row_0/B:col10/1731916444895/Put/seqid=0 2024-11-18T07:54:06,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/1c8b1dccba004854a8f0b83339933684 is 50, key is test_row_0/A:col10/1731916446108/Put/seqid=0 2024-11-18T07:54:06,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742276_1452 (size=12254) 2024-11-18T07:54:06,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742277_1453 (size=12254) 2024-11-18T07:54:06,669 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/2c6038cbcdc84db3ad9522613f7f764f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2c6038cbcdc84db3ad9522613f7f764f 2024-11-18T07:54:06,674 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/B of deeef93d343bf8d154ce5359480346cc into 2c6038cbcdc84db3ad9522613f7f764f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:06,674 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:06,674 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/B, priority=13, startTime=1731916446555; duration=0sec 2024-11-18T07:54:06,674 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:06,674 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:06,675 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:06,676 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:06,676 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/C is initiating minor compaction (all files) 2024-11-18T07:54:06,676 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/C in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:06,676 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/e974f635388d4402a3a6dd9fabfacfc9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/82cd47cd819a48a2891e1f556337c824, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/261d30c401564939b23fa7b7f72197fa] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=35.8 K 2024-11-18T07:54:06,676 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting e974f635388d4402a3a6dd9fabfacfc9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731916443005 2024-11-18T07:54:06,677 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 82cd47cd819a48a2891e1f556337c824, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731916443696 2024-11-18T07:54:06,677 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 261d30c401564939b23fa7b7f72197fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731916444890 2024-11-18T07:54:06,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742278_1454 (size=12151) 2024-11-18T07:54:06,691 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/1c8b1dccba004854a8f0b83339933684 2024-11-18T07:54:06,698 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#C#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:06,699 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/4679df8767554b9592723a25d74423b4 is 50, key is test_row_0/C:col10/1731916444895/Put/seqid=0 2024-11-18T07:54:06,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/577085ad0d8b43dcb789b5db876521e3 is 50, key is test_row_0/B:col10/1731916446108/Put/seqid=0 2024-11-18T07:54:06,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742279_1455 (size=12254) 2024-11-18T07:54:06,722 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/4679df8767554b9592723a25d74423b4 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/4679df8767554b9592723a25d74423b4 2024-11-18T07:54:06,727 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/C of deeef93d343bf8d154ce5359480346cc into 4679df8767554b9592723a25d74423b4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:06,727 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:06,727 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/C, priority=13, startTime=1731916446556; duration=0sec 2024-11-18T07:54:06,727 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:06,727 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:C 2024-11-18T07:54:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T07:54:06,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:06,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742280_1456 (size=12151) 2024-11-18T07:54:06,750 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/577085ad0d8b43dcb789b5db876521e3 2024-11-18T07:54:06,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/f825e5e5b3e94846a4ef0fcbcf5aa608 is 50, key is test_row_0/C:col10/1731916446108/Put/seqid=0 2024-11-18T07:54:06,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742281_1457 (size=12151) 2024-11-18T07:54:06,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916506782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916506788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916506789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916506791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916506897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916506908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916506908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:06,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916506908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:06,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:06,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T07:54:07,069 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/ba7522c7aafb42739561bd2e62c46db3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/ba7522c7aafb42739561bd2e62c46db3 2024-11-18T07:54:07,075 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/A of deeef93d343bf8d154ce5359480346cc into ba7522c7aafb42739561bd2e62c46db3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:07,075 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:07,075 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/A, priority=13, startTime=1731916446555; duration=0sec 2024-11-18T07:54:07,076 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:07,076 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:A 2024-11-18T07:54:07,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916507108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916507128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916507128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916507128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,172 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/f825e5e5b3e94846a4ef0fcbcf5aa608 2024-11-18T07:54:07,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/1c8b1dccba004854a8f0b83339933684 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1c8b1dccba004854a8f0b83339933684 2024-11-18T07:54:07,182 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1c8b1dccba004854a8f0b83339933684, entries=150, sequenceid=185, filesize=11.9 K 2024-11-18T07:54:07,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/577085ad0d8b43dcb789b5db876521e3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/577085ad0d8b43dcb789b5db876521e3 2024-11-18T07:54:07,190 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/577085ad0d8b43dcb789b5db876521e3, entries=150, sequenceid=185, filesize=11.9 K 2024-11-18T07:54:07,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/f825e5e5b3e94846a4ef0fcbcf5aa608 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/f825e5e5b3e94846a4ef0fcbcf5aa608 2024-11-18T07:54:07,199 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/f825e5e5b3e94846a4ef0fcbcf5aa608, entries=150, sequenceid=185, filesize=11.9 K 2024-11-18T07:54:07,200 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for deeef93d343bf8d154ce5359480346cc in 627ms, sequenceid=185, compaction requested=false 2024-11-18T07:54:07,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:07,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:07,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-18T07:54:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-18T07:54:07,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-18T07:54:07,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 783 msec 2024-11-18T07:54:07,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 789 msec 2024-11-18T07:54:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:07,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T07:54:07,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:07,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:07,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:07,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:07,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:07,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:07,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/5c1b5f639d5848e2ab1168733ae29901 is 50, key is test_row_0/A:col10/1731916446789/Put/seqid=0 2024-11-18T07:54:07,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742282_1458 (size=14541) 2024-11-18T07:54:07,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/5c1b5f639d5848e2ab1168733ae29901 2024-11-18T07:54:07,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/7d00eaed894246bf940988256f1ef518 is 50, key is test_row_0/B:col10/1731916446789/Put/seqid=0 2024-11-18T07:54:07,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742283_1459 (size=12151) 2024-11-18T07:54:07,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/7d00eaed894246bf940988256f1ef518 2024-11-18T07:54:07,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916507467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/aa8d3ec975c74924a04d3ff119ea1d69 is 50, key is test_row_0/C:col10/1731916446789/Put/seqid=0 2024-11-18T07:54:07,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916507483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916507484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916507486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742284_1460 (size=12151) 2024-11-18T07:54:07,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/aa8d3ec975c74924a04d3ff119ea1d69 2024-11-18T07:54:07,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/5c1b5f639d5848e2ab1168733ae29901 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/5c1b5f639d5848e2ab1168733ae29901 2024-11-18T07:54:07,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/5c1b5f639d5848e2ab1168733ae29901, entries=200, sequenceid=204, filesize=14.2 K 2024-11-18T07:54:07,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/7d00eaed894246bf940988256f1ef518 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/7d00eaed894246bf940988256f1ef518 2024-11-18T07:54:07,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/7d00eaed894246bf940988256f1ef518, entries=150, sequenceid=204, filesize=11.9 K 2024-11-18T07:54:07,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/aa8d3ec975c74924a04d3ff119ea1d69 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/aa8d3ec975c74924a04d3ff119ea1d69 2024-11-18T07:54:07,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/aa8d3ec975c74924a04d3ff119ea1d69, entries=150, sequenceid=204, filesize=11.9 K 2024-11-18T07:54:07,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for deeef93d343bf8d154ce5359480346cc in 103ms, sequenceid=204, compaction requested=true 2024-11-18T07:54:07,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:07,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:07,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:07,540 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:07,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:07,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:07,540 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:07,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:07,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:07,541 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36556 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:07,541 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38946 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:07,541 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/B is initiating minor compaction (all files) 2024-11-18T07:54:07,541 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/A is initiating minor compaction (all files) 2024-11-18T07:54:07,541 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/B in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:07,541 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/A in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:07,542 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2c6038cbcdc84db3ad9522613f7f764f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/577085ad0d8b43dcb789b5db876521e3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/7d00eaed894246bf940988256f1ef518] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=35.7 K 2024-11-18T07:54:07,542 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/ba7522c7aafb42739561bd2e62c46db3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1c8b1dccba004854a8f0b83339933684, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/5c1b5f639d5848e2ab1168733ae29901] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=38.0 K 2024-11-18T07:54:07,542 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c6038cbcdc84db3ad9522613f7f764f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731916444890 2024-11-18T07:54:07,542 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba7522c7aafb42739561bd2e62c46db3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731916444890 2024-11-18T07:54:07,543 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 577085ad0d8b43dcb789b5db876521e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731916446095 2024-11-18T07:54:07,543 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c8b1dccba004854a8f0b83339933684, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731916446095 2024-11-18T07:54:07,543 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d00eaed894246bf940988256f1ef518, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731916446780 2024-11-18T07:54:07,543 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c1b5f639d5848e2ab1168733ae29901, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731916446772 2024-11-18T07:54:07,551 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#B#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:07,551 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/54a7da08a527410cb5fbab4ab664f1ee is 50, key is test_row_0/B:col10/1731916446789/Put/seqid=0 2024-11-18T07:54:07,552 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#A#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:07,552 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/c20880e01a67440fbe20f7902727e4dc is 50, key is test_row_0/A:col10/1731916446789/Put/seqid=0 2024-11-18T07:54:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T07:54:07,558 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:07,559 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:54:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-18T07:54:07,561 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:54:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T07:54:07,561 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:54:07,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:07,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742285_1461 (size=12357) 2024-11-18T07:54:07,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742286_1462 (size=12357) 2024-11-18T07:54:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:07,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T07:54:07,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:07,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:07,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:07,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:07,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:07,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:07,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/951d73d9ede64750829dd7824ebcd16f is 50, key is test_row_0/A:col10/1731916447483/Put/seqid=0 2024-11-18T07:54:07,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742287_1463 (size=14541) 2024-11-18T07:54:07,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916507636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916507637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916507653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T07:54:07,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916507655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T07:54:07,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:07,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:07,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:07,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:07,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:07,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:07,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916507758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916507758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916507768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916507778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T07:54:07,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:07,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:07,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:07,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:07,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:07,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T07:54:07,971 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/54a7da08a527410cb5fbab4ab664f1ee as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/54a7da08a527410cb5fbab4ab664f1ee 2024-11-18T07:54:07,972 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/c20880e01a67440fbe20f7902727e4dc as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c20880e01a67440fbe20f7902727e4dc 2024-11-18T07:54:07,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916507968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916507968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:07,990 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/A of deeef93d343bf8d154ce5359480346cc into c20880e01a67440fbe20f7902727e4dc(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:07,990 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/B of deeef93d343bf8d154ce5359480346cc into 54a7da08a527410cb5fbab4ab664f1ee(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:07,990 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:07,990 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:07,990 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/A, priority=13, startTime=1731916447540; duration=0sec 2024-11-18T07:54:07,990 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/B, priority=13, startTime=1731916447540; duration=0sec 2024-11-18T07:54:07,990 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:07,990 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:A 2024-11-18T07:54:07,990 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:07,990 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:07,990 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:07,991 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36556 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:07,991 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/C is initiating minor compaction (all files) 2024-11-18T07:54:07,991 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/C in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:07,991 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/4679df8767554b9592723a25d74423b4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/f825e5e5b3e94846a4ef0fcbcf5aa608, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/aa8d3ec975c74924a04d3ff119ea1d69] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=35.7 K 2024-11-18T07:54:07,991 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4679df8767554b9592723a25d74423b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731916444890 2024-11-18T07:54:07,992 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f825e5e5b3e94846a4ef0fcbcf5aa608, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731916446095 2024-11-18T07:54:07,992 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa8d3ec975c74924a04d3ff119ea1d69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731916446780 2024-11-18T07:54:07,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:07,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916507988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:08,004 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#C#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:08,005 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/b057635b4942436aa3898121c4ecd213 is 50, key is test_row_0/C:col10/1731916446789/Put/seqid=0 2024-11-18T07:54:08,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:08,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916507998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:08,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T07:54:08,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:08,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742288_1464 (size=12357) 2024-11-18T07:54:08,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:08,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:08,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:08,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:08,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:08,027 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/b057635b4942436aa3898121c4ecd213 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/b057635b4942436aa3898121c4ecd213 2024-11-18T07:54:08,033 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/C of deeef93d343bf8d154ce5359480346cc into b057635b4942436aa3898121c4ecd213(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:08,033 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:08,033 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/C, priority=13, startTime=1731916447540; duration=0sec 2024-11-18T07:54:08,033 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:08,033 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:C 2024-11-18T07:54:08,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/951d73d9ede64750829dd7824ebcd16f 2024-11-18T07:54:08,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/f9dd8da4f8bf4eb89ccff3f6c9299422 is 50, key is test_row_0/B:col10/1731916447483/Put/seqid=0 2024-11-18T07:54:08,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742289_1465 (size=12151) 2024-11-18T07:54:08,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/f9dd8da4f8bf4eb89ccff3f6c9299422 2024-11-18T07:54:08,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/cc9afd305d7848829d473e6f222807af is 50, key is test_row_0/C:col10/1731916447483/Put/seqid=0 2024-11-18T07:54:08,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742290_1466 (size=12151) 2024-11-18T07:54:08,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T07:54:08,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:08,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:08,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:08,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:08,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:08,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:08,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T07:54:08,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916508288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:08,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916508298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:08,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916508309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:08,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916508318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:08,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T07:54:08,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:08,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:08,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:08,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:08,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:08,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:08,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T07:54:08,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:08,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:08,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:08,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:08,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:08,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/cc9afd305d7848829d473e6f222807af 2024-11-18T07:54:08,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/951d73d9ede64750829dd7824ebcd16f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/951d73d9ede64750829dd7824ebcd16f 2024-11-18T07:54:08,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/951d73d9ede64750829dd7824ebcd16f, entries=200, sequenceid=221, filesize=14.2 K 2024-11-18T07:54:08,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/f9dd8da4f8bf4eb89ccff3f6c9299422 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/f9dd8da4f8bf4eb89ccff3f6c9299422 2024-11-18T07:54:08,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/f9dd8da4f8bf4eb89ccff3f6c9299422, entries=150, sequenceid=221, filesize=11.9 K 2024-11-18T07:54:08,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/cc9afd305d7848829d473e6f222807af as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/cc9afd305d7848829d473e6f222807af 2024-11-18T07:54:08,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/cc9afd305d7848829d473e6f222807af, entries=150, sequenceid=221, filesize=11.9 K 2024-11-18T07:54:08,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for deeef93d343bf8d154ce5359480346cc in 934ms, sequenceid=221, compaction requested=false 2024-11-18T07:54:08,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:08,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T07:54:08,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:08,637 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T07:54:08,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:08,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:08,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:08,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:08,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:08,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:08,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/4465e0e5f5c042fcb31a9724a838c3d2 is 50, key is test_row_0/A:col10/1731916447652/Put/seqid=0 2024-11-18T07:54:08,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742291_1467 (size=12151) 2024-11-18T07:54:08,695 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/4465e0e5f5c042fcb31a9724a838c3d2 2024-11-18T07:54:08,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T07:54:08,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/8466a9553f1d4a85bbfb99cb2fdb5532 is 50, key is test_row_0/B:col10/1731916447652/Put/seqid=0 2024-11-18T07:54:08,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742292_1468 (size=12151) 2024-11-18T07:54:08,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916508870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:08,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916508872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:08,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916508878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:08,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:08,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916508882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:08,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:08,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916508988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916508998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916509008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916509008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,166 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/8466a9553f1d4a85bbfb99cb2fdb5532 2024-11-18T07:54:09,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/d3ffc0d780b04221ac9933680f02fd11 is 50, key is test_row_0/C:col10/1731916447652/Put/seqid=0 2024-11-18T07:54:09,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916509208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742293_1469 (size=12151) 2024-11-18T07:54:09,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916509218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,236 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/d3ffc0d780b04221ac9933680f02fd11 2024-11-18T07:54:09,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/4465e0e5f5c042fcb31a9724a838c3d2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/4465e0e5f5c042fcb31a9724a838c3d2 2024-11-18T07:54:09,246 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/4465e0e5f5c042fcb31a9724a838c3d2, entries=150, sequenceid=240, filesize=11.9 K 2024-11-18T07:54:09,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/8466a9553f1d4a85bbfb99cb2fdb5532 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/8466a9553f1d4a85bbfb99cb2fdb5532 2024-11-18T07:54:09,251 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/8466a9553f1d4a85bbfb99cb2fdb5532, entries=150, sequenceid=240, filesize=11.9 K 2024-11-18T07:54:09,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/d3ffc0d780b04221ac9933680f02fd11 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d3ffc0d780b04221ac9933680f02fd11 2024-11-18T07:54:09,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916509238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916509238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,256 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d3ffc0d780b04221ac9933680f02fd11, entries=150, sequenceid=240, filesize=11.9 K 2024-11-18T07:54:09,257 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for deeef93d343bf8d154ce5359480346cc in 620ms, sequenceid=240, compaction requested=true 2024-11-18T07:54:09,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:09,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:09,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-18T07:54:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-18T07:54:09,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-18T07:54:09,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6970 sec 2024-11-18T07:54:09,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.7020 sec 2024-11-18T07:54:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:09,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T07:54:09,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:09,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:09,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:09,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:09,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:09,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:09,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/c60a75fa49c741a696c91b5a268d451b is 50, key is test_row_0/A:col10/1731916448856/Put/seqid=0 2024-11-18T07:54:09,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742294_1470 (size=14591) 2024-11-18T07:54:09,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/c60a75fa49c741a696c91b5a268d451b 2024-11-18T07:54:09,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/778e16f64eba4864906db41e9de2e297 is 50, key is test_row_0/B:col10/1731916448856/Put/seqid=0 2024-11-18T07:54:09,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742295_1471 (size=12201) 2024-11-18T07:54:09,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916509578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916509579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916509583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916509590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T07:54:09,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,708 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916509697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,709 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:54:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-18T07:54:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T07:54:09,711 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:54:09,711 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:54:09,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:09,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916509708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916509708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916509708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T07:54:09,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T07:54:09,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:09,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:09,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:09,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:09,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:09,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916509918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916509928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916509928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916509928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:09,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/778e16f64eba4864906db41e9de2e297 2024-11-18T07:54:09,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/494a8b801b2d473e988e4e2c888c3fd3 is 50, key is test_row_0/C:col10/1731916448856/Put/seqid=0 2024-11-18T07:54:09,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742296_1472 (size=12201) 2024-11-18T07:54:10,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T07:54:10,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:10,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:10,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:10,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:10,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T07:54:10,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T07:54:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:10,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:10,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916510239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:10,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916510239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:10,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916510239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:10,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916510239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:10,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T07:54:10,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:10,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:10,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:10,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:10,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T07:54:10,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/494a8b801b2d473e988e4e2c888c3fd3 2024-11-18T07:54:10,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/c60a75fa49c741a696c91b5a268d451b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c60a75fa49c741a696c91b5a268d451b 2024-11-18T07:54:10,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c60a75fa49c741a696c91b5a268d451b, entries=200, sequenceid=257, filesize=14.2 K 2024-11-18T07:54:10,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/778e16f64eba4864906db41e9de2e297 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/778e16f64eba4864906db41e9de2e297 2024-11-18T07:54:10,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/778e16f64eba4864906db41e9de2e297, entries=150, sequenceid=257, filesize=11.9 K 2024-11-18T07:54:10,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/494a8b801b2d473e988e4e2c888c3fd3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/494a8b801b2d473e988e4e2c888c3fd3 2024-11-18T07:54:10,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/494a8b801b2d473e988e4e2c888c3fd3, entries=150, sequenceid=257, filesize=11.9 K 2024-11-18T07:54:10,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for deeef93d343bf8d154ce5359480346cc in 869ms, sequenceid=257, compaction requested=true 2024-11-18T07:54:10,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:10,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:10,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:10,395 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:54:10,395 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:54:10,396 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53640 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:54:10,396 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/A is initiating minor compaction (all files) 2024-11-18T07:54:10,396 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/A in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:10,396 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c20880e01a67440fbe20f7902727e4dc, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/951d73d9ede64750829dd7824ebcd16f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/4465e0e5f5c042fcb31a9724a838c3d2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c60a75fa49c741a696c91b5a268d451b] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=52.4 K 2024-11-18T07:54:10,397 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c20880e01a67440fbe20f7902727e4dc, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731916446780 2024-11-18T07:54:10,397 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 951d73d9ede64750829dd7824ebcd16f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731916447465 2024-11-18T07:54:10,397 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48860 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:54:10,397 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/B is initiating minor compaction (all files) 2024-11-18T07:54:10,398 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/B in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:10,398 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/54a7da08a527410cb5fbab4ab664f1ee, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/f9dd8da4f8bf4eb89ccff3f6c9299422, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/8466a9553f1d4a85bbfb99cb2fdb5532, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/778e16f64eba4864906db41e9de2e297] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=47.7 K 2024-11-18T07:54:10,398 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4465e0e5f5c042fcb31a9724a838c3d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731916447633 2024-11-18T07:54:10,399 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 54a7da08a527410cb5fbab4ab664f1ee, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731916446780 2024-11-18T07:54:10,399 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c60a75fa49c741a696c91b5a268d451b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731916448856 2024-11-18T07:54:10,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:10,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:10,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:10,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:10,399 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting f9dd8da4f8bf4eb89ccff3f6c9299422, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731916447481 2024-11-18T07:54:10,400 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8466a9553f1d4a85bbfb99cb2fdb5532, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731916447633 2024-11-18T07:54:10,400 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 778e16f64eba4864906db41e9de2e297, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731916448856 2024-11-18T07:54:10,410 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#A#compaction#399 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:10,410 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/6d6a57c5773d4f3c87a85b33d9b50c19 is 50, key is test_row_0/A:col10/1731916448856/Put/seqid=0 2024-11-18T07:54:10,414 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#B#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:10,414 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/fabd0bf0185048b5a3f54225550bdd9d is 50, key is test_row_0/B:col10/1731916448856/Put/seqid=0 2024-11-18T07:54:10,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742298_1474 (size=12543) 2024-11-18T07:54:10,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742297_1473 (size=12543) 2024-11-18T07:54:10,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T07:54:10,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:10,481 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T07:54:10,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:10,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:10,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:10,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:10,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:10,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:10,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/fcf4f3b4160642028827f4e5d18254ab is 50, key is test_row_0/A:col10/1731916449581/Put/seqid=0 2024-11-18T07:54:10,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742299_1475 (size=12301) 2024-11-18T07:54:10,503 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/fcf4f3b4160642028827f4e5d18254ab 2024-11-18T07:54:10,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/2ada0d0398c14d0283225370f349d331 is 50, key is test_row_0/B:col10/1731916449581/Put/seqid=0 2024-11-18T07:54:10,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742300_1476 (size=12301) 2024-11-18T07:54:10,544 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/2ada0d0398c14d0283225370f349d331 2024-11-18T07:54:10,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/27b033c1914246aabacae8dfd4fc5cf4 is 50, key is test_row_0/C:col10/1731916449581/Put/seqid=0 2024-11-18T07:54:10,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742301_1477 (size=12301) 2024-11-18T07:54:10,576 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/27b033c1914246aabacae8dfd4fc5cf4 2024-11-18T07:54:10,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/fcf4f3b4160642028827f4e5d18254ab as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/fcf4f3b4160642028827f4e5d18254ab 2024-11-18T07:54:10,597 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/fcf4f3b4160642028827f4e5d18254ab, entries=150, sequenceid=272, filesize=12.0 K 2024-11-18T07:54:10,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/2ada0d0398c14d0283225370f349d331 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2ada0d0398c14d0283225370f349d331 2024-11-18T07:54:10,608 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2ada0d0398c14d0283225370f349d331, entries=150, sequenceid=272, filesize=12.0 K 2024-11-18T07:54:10,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/27b033c1914246aabacae8dfd4fc5cf4 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/27b033c1914246aabacae8dfd4fc5cf4 2024-11-18T07:54:10,613 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/27b033c1914246aabacae8dfd4fc5cf4, entries=150, sequenceid=272, filesize=12.0 K 2024-11-18T07:54:10,614 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for deeef93d343bf8d154ce5359480346cc in 133ms, sequenceid=272, compaction requested=true 2024-11-18T07:54:10,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:10,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:10,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-18T07:54:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-18T07:54:10,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-18T07:54:10,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 905 msec 2024-11-18T07:54:10,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 909 msec 2024-11-18T07:54:10,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:10,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:54:10,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:10,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:10,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:10,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:10,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:10,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:10,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/1ac62f6ccad24e09b856358b0ddbbfc4 is 50, key is test_row_0/A:col10/1731916450760/Put/seqid=0 2024-11-18T07:54:10,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742302_1478 (size=19621) 2024-11-18T07:54:10,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/1ac62f6ccad24e09b856358b0ddbbfc4 2024-11-18T07:54:10,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/9eaba1dc858943f28211e6c205518f0b is 50, key is test_row_0/B:col10/1731916450760/Put/seqid=0 2024-11-18T07:54:10,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742303_1479 (size=12301) 2024-11-18T07:54:10,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/9eaba1dc858943f28211e6c205518f0b 2024-11-18T07:54:10,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/c1ab4544b9fd4b08941e41b1732800e9 is 50, key is test_row_0/C:col10/1731916450760/Put/seqid=0 2024-11-18T07:54:10,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916450783, max timestamp=1731916450831Num uniques 859; 2024-11-18T07:54:10,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916450783, max timestamp=1731916450831Num uniques 859; 2024-11-18T07:54:10,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:10,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916450783, max timestamp=1731916450831Num uniques 859; 2024-11-18T07:54:10,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:10,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:10,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742304_1480 (size=12301) 2024-11-18T07:54:10,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/c1ab4544b9fd4b08941e41b1732800e9 2024-11-18T07:54:10,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/1ac62f6ccad24e09b856358b0ddbbfc4 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1ac62f6ccad24e09b856358b0ddbbfc4 2024-11-18T07:54:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T07:54:10,849 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:10,849 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:54:10,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1ac62f6ccad24e09b856358b0ddbbfc4, entries=300, sequenceid=282, filesize=19.2 K 2024-11-18T07:54:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-18T07:54:10,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/9eaba1dc858943f28211e6c205518f0b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/9eaba1dc858943f28211e6c205518f0b 2024-11-18T07:54:10,852 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:54:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T07:54:10,853 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:54:10,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:10,855 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/fabd0bf0185048b5a3f54225550bdd9d as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fabd0bf0185048b5a3f54225550bdd9d 2024-11-18T07:54:10,858 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/6d6a57c5773d4f3c87a85b33d9b50c19 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/6d6a57c5773d4f3c87a85b33d9b50c19 2024-11-18T07:54:10,865 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in deeef93d343bf8d154ce5359480346cc/B of deeef93d343bf8d154ce5359480346cc into fabd0bf0185048b5a3f54225550bdd9d(size=12.2 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:10,865 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:10,865 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/B, priority=12, startTime=1731916450395; duration=0sec 2024-11-18T07:54:10,865 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:10,865 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:10,865 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-18T07:54:10,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/9eaba1dc858943f28211e6c205518f0b, entries=150, sequenceid=282, filesize=12.0 K 2024-11-18T07:54:10,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/c1ab4544b9fd4b08941e41b1732800e9 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/c1ab4544b9fd4b08941e41b1732800e9 2024-11-18T07:54:10,868 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61161 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-18T07:54:10,868 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/C is initiating minor compaction (all files) 2024-11-18T07:54:10,868 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in deeef93d343bf8d154ce5359480346cc/A of deeef93d343bf8d154ce5359480346cc into 6d6a57c5773d4f3c87a85b33d9b50c19(size=12.2 K), total size for store is 43.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:10,868 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:10,868 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/C in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:10,868 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/A, priority=12, startTime=1731916450395; duration=0sec 2024-11-18T07:54:10,868 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/b057635b4942436aa3898121c4ecd213, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/cc9afd305d7848829d473e6f222807af, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d3ffc0d780b04221ac9933680f02fd11, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/494a8b801b2d473e988e4e2c888c3fd3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/27b033c1914246aabacae8dfd4fc5cf4] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=59.7 K 2024-11-18T07:54:10,868 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:10,869 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:A 2024-11-18T07:54:10,869 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting b057635b4942436aa3898121c4ecd213, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731916446780 2024-11-18T07:54:10,869 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting cc9afd305d7848829d473e6f222807af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731916447481 2024-11-18T07:54:10,869 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting d3ffc0d780b04221ac9933680f02fd11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731916447633 2024-11-18T07:54:10,870 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 494a8b801b2d473e988e4e2c888c3fd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731916448856 2024-11-18T07:54:10,870 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 27b033c1914246aabacae8dfd4fc5cf4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731916449571 2024-11-18T07:54:10,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/c1ab4544b9fd4b08941e41b1732800e9, entries=150, sequenceid=282, filesize=12.0 K 2024-11-18T07:54:10,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=73.80 KB/75570 for deeef93d343bf8d154ce5359480346cc in 109ms, sequenceid=282, compaction requested=true 2024-11-18T07:54:10,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:10,874 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:10,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:A, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:10,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:10,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:B, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:10,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:10,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:10,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:54:10,875 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:10,876 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/A is initiating minor compaction (all files) 2024-11-18T07:54:10,876 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/A in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:10,876 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/6d6a57c5773d4f3c87a85b33d9b50c19, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/fcf4f3b4160642028827f4e5d18254ab, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1ac62f6ccad24e09b856358b0ddbbfc4] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=43.4 K 2024-11-18T07:54:10,876 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d6a57c5773d4f3c87a85b33d9b50c19, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731916448856 2024-11-18T07:54:10,876 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcf4f3b4160642028827f4e5d18254ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731916449571 2024-11-18T07:54:10,877 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ac62f6ccad24e09b856358b0ddbbfc4, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731916450751 2024-11-18T07:54:10,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:10,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=80.51 KB heapSize=220.92 KB 2024-11-18T07:54:10,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:10,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:10,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:10,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:10,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:10,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:10,884 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#C#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:10,884 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/392106e24b5b40349d40abd8acce5d99 is 50, key is test_row_0/C:col10/1731916449581/Put/seqid=0 2024-11-18T07:54:10,887 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#A#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:10,888 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/0dbfde4c811247c0a5576ab4848619cb is 50, key is test_row_0/A:col10/1731916450760/Put/seqid=0 2024-11-18T07:54:10,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/c4df6961ffa24226bf967d0545e34c31 is 50, key is test_row_0/A:col10/1731916450872/Put/seqid=0 2024-11-18T07:54:10,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742306_1482 (size=12745) 2024-11-18T07:54:10,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742305_1481 (size=12677) 2024-11-18T07:54:10,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742307_1483 (size=17181) 2024-11-18T07:54:10,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/c4df6961ffa24226bf967d0545e34c31 2024-11-18T07:54:10,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/2b217f65308e4925be92767caa50ca32 is 50, key is test_row_0/B:col10/1731916450872/Put/seqid=0 2024-11-18T07:54:10,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742308_1484 (size=12301) 2024-11-18T07:54:10,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916510935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:10,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916510952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T07:54:10,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916510953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:10,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916510954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:11,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T07:54:11,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:11,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:11,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:11,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916511058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:11,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916511058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:11,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916511078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:11,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916511078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:11,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T07:54:11,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:11,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:11,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:11,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T07:54:11,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916511278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:11,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:11,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916511280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:11,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:11,306 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/0dbfde4c811247c0a5576ab4848619cb as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/0dbfde4c811247c0a5576ab4848619cb 2024-11-18T07:54:11,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T07:54:11,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:11,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916511298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:11,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:11,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:11,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916511299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,315 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/392106e24b5b40349d40abd8acce5d99 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/392106e24b5b40349d40abd8acce5d99 2024-11-18T07:54:11,322 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/A of deeef93d343bf8d154ce5359480346cc into 0dbfde4c811247c0a5576ab4848619cb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:11,322 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in deeef93d343bf8d154ce5359480346cc/C of deeef93d343bf8d154ce5359480346cc into 392106e24b5b40349d40abd8acce5d99(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:11,322 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:11,322 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:11,322 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/A, priority=13, startTime=1731916450873; duration=0sec 2024-11-18T07:54:11,322 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/C, priority=11, startTime=1731916450399; duration=0sec 2024-11-18T07:54:11,322 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:54:11,322 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:A 2024-11-18T07:54:11,322 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:54:11,322 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:C 2024-11-18T07:54:11,322 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:C 2024-11-18T07:54:11,322 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-18T07:54:11,322 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:11,323 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T07:54:11,323 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37145 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:11,323 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/B is initiating minor compaction (all files) 2024-11-18T07:54:11,323 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T07:54:11,323 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/B in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:11,323 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. because compaction request was cancelled 2024-11-18T07:54:11,323 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:C 2024-11-18T07:54:11,323 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fabd0bf0185048b5a3f54225550bdd9d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2ada0d0398c14d0283225370f349d331, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/9eaba1dc858943f28211e6c205518f0b] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=36.3 K 2024-11-18T07:54:11,324 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting fabd0bf0185048b5a3f54225550bdd9d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731916448856 2024-11-18T07:54:11,324 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ada0d0398c14d0283225370f349d331, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731916449571 2024-11-18T07:54:11,324 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9eaba1dc858943f28211e6c205518f0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731916450760 2024-11-18T07:54:11,333 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:11,333 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/8d9b4311c6ca44d79017b31a97a14a0f is 50, key is test_row_0/B:col10/1731916450760/Put/seqid=0 2024-11-18T07:54:11,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/2b217f65308e4925be92767caa50ca32 2024-11-18T07:54:11,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742309_1485 (size=12745) 2024-11-18T07:54:11,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/12e6664c4bb44a5e95270d6b639c4da2 is 50, key is test_row_0/C:col10/1731916450872/Put/seqid=0 2024-11-18T07:54:11,349 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/8d9b4311c6ca44d79017b31a97a14a0f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/8d9b4311c6ca44d79017b31a97a14a0f 2024-11-18T07:54:11,354 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/B of deeef93d343bf8d154ce5359480346cc into 8d9b4311c6ca44d79017b31a97a14a0f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:11,354 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:11,354 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/B, priority=13, startTime=1731916450874; duration=0sec 2024-11-18T07:54:11,354 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:11,354 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:11,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742310_1486 (size=12301) 2024-11-18T07:54:11,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T07:54:11,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:11,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:11,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:11,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T07:54:11,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:11,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916511598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:11,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916511599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:11,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:11,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:11,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916511618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:11,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:11,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916511618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:11,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:11,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T07:54:11,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:11,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:11,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:11,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/12e6664c4bb44a5e95270d6b639c4da2 2024-11-18T07:54:11,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/c4df6961ffa24226bf967d0545e34c31 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c4df6961ffa24226bf967d0545e34c31 2024-11-18T07:54:11,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T07:54:11,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:11,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:11,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:11,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:11,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c4df6961ffa24226bf967d0545e34c31, entries=250, sequenceid=312, filesize=16.8 K 2024-11-18T07:54:11,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/2b217f65308e4925be92767caa50ca32 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2b217f65308e4925be92767caa50ca32 2024-11-18T07:54:11,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2b217f65308e4925be92767caa50ca32, entries=150, sequenceid=312, filesize=12.0 K 2024-11-18T07:54:11,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/12e6664c4bb44a5e95270d6b639c4da2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/12e6664c4bb44a5e95270d6b639c4da2 2024-11-18T07:54:11,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/12e6664c4bb44a5e95270d6b639c4da2, entries=150, sequenceid=312, filesize=12.0 K 2024-11-18T07:54:11,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~220.88 KB/226176, currentSize=107.34 KB/109920 for deeef93d343bf8d154ce5359480346cc in 942ms, sequenceid=312, compaction requested=true 2024-11-18T07:54:11,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:11,826 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-18T07:54:11,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:11,826 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T07:54:11,826 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T07:54:11,826 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. because compaction request was cancelled 2024-11-18T07:54:11,826 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:A 2024-11-18T07:54:11,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:11,827 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-18T07:54:11,827 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T07:54:11,827 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T07:54:11,827 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. because compaction request was cancelled 2024-11-18T07:54:11,827 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:11,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:B, priority=-2147483648, current under compaction store size is 0 2024-11-18T07:54:11,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:11,828 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:11,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:C, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:11,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:11,829 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:11,829 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/C is initiating minor compaction (all files) 2024-11-18T07:54:11,829 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/C in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:11,829 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/392106e24b5b40349d40abd8acce5d99, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/c1ab4544b9fd4b08941e41b1732800e9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/12e6664c4bb44a5e95270d6b639c4da2] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=36.4 K 2024-11-18T07:54:11,829 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 392106e24b5b40349d40abd8acce5d99, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731916449571 2024-11-18T07:54:11,830 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1ab4544b9fd4b08941e41b1732800e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731916450760 2024-11-18T07:54:11,830 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12e6664c4bb44a5e95270d6b639c4da2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731916450862 2024-11-18T07:54:11,838 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#C#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:11,838 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/fb83198d6ac0402b94bd19384f8bdc94 is 50, key is test_row_0/C:col10/1731916450872/Put/seqid=0 2024-11-18T07:54:11,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742311_1487 (size=12779) 2024-11-18T07:54:11,875 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/fb83198d6ac0402b94bd19384f8bdc94 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/fb83198d6ac0402b94bd19384f8bdc94 2024-11-18T07:54:11,884 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/C of deeef93d343bf8d154ce5359480346cc into fb83198d6ac0402b94bd19384f8bdc94(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:11,884 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:11,884 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/C, priority=13, startTime=1731916451828; duration=0sec 2024-11-18T07:54:11,884 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:11,884 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:C 2024-11-18T07:54:11,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T07:54:11,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:11,959 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T07:54:11,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:11,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:11,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:11,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:11,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:11,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T07:54:11,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/825f39eb9a454b82a498d4c85c3d1f02 is 50, key is test_row_0/A:col10/1731916450952/Put/seqid=0 2024-11-18T07:54:12,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742312_1488 (size=12301) 2024-11-18T07:54:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:12,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:12,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916512158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:12,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:12,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916512160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:12,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:12,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916512168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:12,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916512171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:12,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916512278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:12,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916512279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:12,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916512293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:12,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916512298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:12,439 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/825f39eb9a454b82a498d4c85c3d1f02 2024-11-18T07:54:12,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/4c984d6c28dc4b6bbf8e0522c93ed0f5 is 50, key is test_row_0/B:col10/1731916450952/Put/seqid=0 2024-11-18T07:54:12,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742313_1489 (size=12301) 2024-11-18T07:54:12,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916512499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:12,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916512504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:12,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916512510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:12,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:12,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916512518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:12,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:12,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916512818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:12,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916512828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:12,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916512838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:12,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:12,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916512838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:12,913 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/4c984d6c28dc4b6bbf8e0522c93ed0f5 2024-11-18T07:54:12,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/8640b5df872e4616b52f469cc978c0db is 50, key is test_row_0/C:col10/1731916450952/Put/seqid=0 2024-11-18T07:54:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T07:54:13,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742314_1490 (size=12301) 2024-11-18T07:54:13,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916513338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:13,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916513358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:13,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916513363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:13,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:13,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:13,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:13,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916513365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:13,406 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/8640b5df872e4616b52f469cc978c0db 2024-11-18T07:54:13,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/825f39eb9a454b82a498d4c85c3d1f02 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/825f39eb9a454b82a498d4c85c3d1f02 2024-11-18T07:54:13,429 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/825f39eb9a454b82a498d4c85c3d1f02, entries=150, sequenceid=335, filesize=12.0 K 2024-11-18T07:54:13,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/4c984d6c28dc4b6bbf8e0522c93ed0f5 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/4c984d6c28dc4b6bbf8e0522c93ed0f5 2024-11-18T07:54:13,451 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/4c984d6c28dc4b6bbf8e0522c93ed0f5, entries=150, sequenceid=335, filesize=12.0 K 2024-11-18T07:54:13,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/8640b5df872e4616b52f469cc978c0db as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/8640b5df872e4616b52f469cc978c0db 2024-11-18T07:54:13,456 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/8640b5df872e4616b52f469cc978c0db, entries=150, sequenceid=335, filesize=12.0 K 2024-11-18T07:54:13,457 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for deeef93d343bf8d154ce5359480346cc in 1499ms, sequenceid=335, compaction requested=true 2024-11-18T07:54:13,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:13,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:13,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-18T07:54:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-18T07:54:13,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-18T07:54:13,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6100 sec 2024-11-18T07:54:13,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.6170 sec 2024-11-18T07:54:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:14,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T07:54:14,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:14,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:14,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:14,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:14,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:14,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:14,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/07f12463ada541cfac2f50a02064c083 is 50, key is test_row_0/A:col10/1731916454366/Put/seqid=0 2024-11-18T07:54:14,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742315_1491 (size=14741) 2024-11-18T07:54:14,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/07f12463ada541cfac2f50a02064c083 2024-11-18T07:54:14,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/fbabb5b09aac420e8ca4b967bef90f3e is 50, key is test_row_0/B:col10/1731916454366/Put/seqid=0 2024-11-18T07:54:14,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742316_1492 (size=12301) 2024-11-18T07:54:14,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916514410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:14,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:14,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916514411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:14,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916514417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:14,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916514423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:14,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916514528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:14,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916514538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:14,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:14,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916514538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:14,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:14,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916514538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:14,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:14,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916514748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:14,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916514758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:14,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916514758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916514758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:14,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/fbabb5b09aac420e8ca4b967bef90f3e 2024-11-18T07:54:14,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/d12b3b8eb5c34ca7b819097c56bf74a5 is 50, key is test_row_0/C:col10/1731916454366/Put/seqid=0 2024-11-18T07:54:14,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742317_1493 (size=12301) 2024-11-18T07:54:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T07:54:15,020 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:15,021 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:54:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-18T07:54:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T07:54:15,028 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:54:15,029 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:54:15,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:15,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916515068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:15,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916515078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916515078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:15,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:15,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916515078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:15,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T07:54:15,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T07:54:15,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:15,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:15,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:15,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:15,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:15,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/d12b3b8eb5c34ca7b819097c56bf74a5 2024-11-18T07:54:15,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/07f12463ada541cfac2f50a02064c083 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/07f12463ada541cfac2f50a02064c083 2024-11-18T07:54:15,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/07f12463ada541cfac2f50a02064c083, entries=200, sequenceid=350, filesize=14.4 K 2024-11-18T07:54:15,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/fbabb5b09aac420e8ca4b967bef90f3e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fbabb5b09aac420e8ca4b967bef90f3e 2024-11-18T07:54:15,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fbabb5b09aac420e8ca4b967bef90f3e, entries=150, sequenceid=350, filesize=12.0 K 2024-11-18T07:54:15,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/d12b3b8eb5c34ca7b819097c56bf74a5 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d12b3b8eb5c34ca7b819097c56bf74a5 2024-11-18T07:54:15,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d12b3b8eb5c34ca7b819097c56bf74a5, entries=150, sequenceid=350, filesize=12.0 K 2024-11-18T07:54:15,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for deeef93d343bf8d154ce5359480346cc in 927ms, sequenceid=350, compaction requested=true 2024-11-18T07:54:15,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:15,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:15,296 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:54:15,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:15,296 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:54:15,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:15,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:15,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:15,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:15,297 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56968 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:54:15,297 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/A is initiating minor compaction (all files) 2024-11-18T07:54:15,297 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/A in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:15,297 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/0dbfde4c811247c0a5576ab4848619cb, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c4df6961ffa24226bf967d0545e34c31, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/825f39eb9a454b82a498d4c85c3d1f02, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/07f12463ada541cfac2f50a02064c083] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=55.6 K 2024-11-18T07:54:15,297 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:54:15,297 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/B is initiating minor compaction (all files) 2024-11-18T07:54:15,297 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/B in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:15,297 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/8d9b4311c6ca44d79017b31a97a14a0f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2b217f65308e4925be92767caa50ca32, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/4c984d6c28dc4b6bbf8e0522c93ed0f5, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fbabb5b09aac420e8ca4b967bef90f3e] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=48.5 K 2024-11-18T07:54:15,298 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dbfde4c811247c0a5576ab4848619cb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731916450760 2024-11-18T07:54:15,298 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d9b4311c6ca44d79017b31a97a14a0f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731916450760 2024-11-18T07:54:15,298 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4df6961ffa24226bf967d0545e34c31, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731916450859 2024-11-18T07:54:15,298 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b217f65308e4925be92767caa50ca32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731916450862 2024-11-18T07:54:15,299 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 825f39eb9a454b82a498d4c85c3d1f02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731916450903 2024-11-18T07:54:15,299 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c984d6c28dc4b6bbf8e0522c93ed0f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731916450903 2024-11-18T07:54:15,299 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07f12463ada541cfac2f50a02064c083, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731916452151 2024-11-18T07:54:15,299 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting fbabb5b09aac420e8ca4b967bef90f3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731916452151 2024-11-18T07:54:15,323 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#B#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:15,324 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/948e60bf8faf4b2cb02ee0da37d1561b is 50, key is test_row_0/B:col10/1731916454366/Put/seqid=0 2024-11-18T07:54:15,337 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#A#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:15,338 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/d397777acf464079b025125fdbe283fe is 50, key is test_row_0/A:col10/1731916454366/Put/seqid=0 2024-11-18T07:54:15,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T07:54:15,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:15,339 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T07:54:15,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:15,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:15,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:15,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:15,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:15,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T07:54:15,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/336e7065bb0945f39dc3f53d551ce380 is 50, key is test_row_0/A:col10/1731916454399/Put/seqid=0 2024-11-18T07:54:15,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742319_1495 (size=12881) 2024-11-18T07:54:15,390 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/d397777acf464079b025125fdbe283fe as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/d397777acf464079b025125fdbe283fe 2024-11-18T07:54:15,396 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in deeef93d343bf8d154ce5359480346cc/A of deeef93d343bf8d154ce5359480346cc into d397777acf464079b025125fdbe283fe(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:15,396 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:15,396 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/A, priority=12, startTime=1731916455296; duration=0sec 2024-11-18T07:54:15,397 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:15,397 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:A 2024-11-18T07:54:15,397 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:15,398 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:15,398 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/C is initiating minor compaction (all files) 2024-11-18T07:54:15,398 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/C in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:15,398 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/fb83198d6ac0402b94bd19384f8bdc94, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/8640b5df872e4616b52f469cc978c0db, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d12b3b8eb5c34ca7b819097c56bf74a5] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=36.5 K 2024-11-18T07:54:15,399 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb83198d6ac0402b94bd19384f8bdc94, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731916450862 2024-11-18T07:54:15,399 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8640b5df872e4616b52f469cc978c0db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731916450903 2024-11-18T07:54:15,399 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d12b3b8eb5c34ca7b819097c56bf74a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731916452151 2024-11-18T07:54:15,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742318_1494 (size=12881) 2024-11-18T07:54:15,418 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/948e60bf8faf4b2cb02ee0da37d1561b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/948e60bf8faf4b2cb02ee0da37d1561b 2024-11-18T07:54:15,423 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in deeef93d343bf8d154ce5359480346cc/B of deeef93d343bf8d154ce5359480346cc into 948e60bf8faf4b2cb02ee0da37d1561b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:15,423 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:15,423 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/B, priority=12, startTime=1731916455296; duration=0sec 2024-11-18T07:54:15,423 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:15,423 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:15,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742320_1496 (size=12301) 2024-11-18T07:54:15,428 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/336e7065bb0945f39dc3f53d551ce380 2024-11-18T07:54:15,433 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#C#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:15,433 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/e7ae50cf8eac4bb4bb731f91e22e9267 is 50, key is test_row_0/C:col10/1731916454366/Put/seqid=0 2024-11-18T07:54:15,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/22bded27da7f4a908695956e2cbadac2 is 50, key is test_row_0/B:col10/1731916454399/Put/seqid=0 2024-11-18T07:54:15,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742321_1497 (size=12881) 2024-11-18T07:54:15,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742322_1498 (size=12301) 2024-11-18T07:54:15,511 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/22bded27da7f4a908695956e2cbadac2 2024-11-18T07:54:15,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/dab5c79e44bb406c91baf5c42c543412 is 50, key is test_row_0/C:col10/1731916454399/Put/seqid=0 2024-11-18T07:54:15,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742323_1499 (size=12301) 2024-11-18T07:54:15,552 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/dab5c79e44bb406c91baf5c42c543412 2024-11-18T07:54:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/336e7065bb0945f39dc3f53d551ce380 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/336e7065bb0945f39dc3f53d551ce380 2024-11-18T07:54:15,570 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/336e7065bb0945f39dc3f53d551ce380, entries=150, sequenceid=367, filesize=12.0 K 2024-11-18T07:54:15,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/22bded27da7f4a908695956e2cbadac2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/22bded27da7f4a908695956e2cbadac2 2024-11-18T07:54:15,577 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/22bded27da7f4a908695956e2cbadac2, entries=150, sequenceid=367, filesize=12.0 K 2024-11-18T07:54:15,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/dab5c79e44bb406c91baf5c42c543412 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/dab5c79e44bb406c91baf5c42c543412 2024-11-18T07:54:15,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:15,587 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/dab5c79e44bb406c91baf5c42c543412, entries=150, sequenceid=367, filesize=12.0 K 2024-11-18T07:54:15,587 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=13.42 KB/13740 for deeef93d343bf8d154ce5359480346cc in 248ms, sequenceid=367, compaction requested=false 2024-11-18T07:54:15,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:15,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:15,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-18T07:54:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-18T07:54:15,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-18T07:54:15,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 560 msec 2024-11-18T07:54:15,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 573 msec 2024-11-18T07:54:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:15,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:54:15,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:15,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:15,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:15,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:15,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:15,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:15,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/f8f7de3b42da40578b2c1c1dcaef6abc is 50, key is test_row_0/A:col10/1731916455604/Put/seqid=0 2024-11-18T07:54:15,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742324_1500 (size=14737) 2024-11-18T07:54:15,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/f8f7de3b42da40578b2c1c1dcaef6abc 2024-11-18T07:54:15,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/d1f9706d18fc4149a88b7734c90eac54 is 50, key is test_row_0/B:col10/1731916455604/Put/seqid=0 2024-11-18T07:54:15,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742325_1501 (size=9857) 2024-11-18T07:54:15,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/d1f9706d18fc4149a88b7734c90eac54 2024-11-18T07:54:15,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/fc8ee9c21a7c4261bf18f86fd1c904f2 is 50, key is test_row_0/C:col10/1731916455604/Put/seqid=0 2024-11-18T07:54:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T07:54:15,663 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:15,664 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:54:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-18T07:54:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T07:54:15,668 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:54:15,669 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:54:15,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:15,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742326_1502 (size=9857) 2024-11-18T07:54:15,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916455648, max timestamp=1731916455681Num uniques 859; 2024-11-18T07:54:15,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:15,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916455648, max timestamp=1731916455681Num uniques 859; 2024-11-18T07:54:15,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:15,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916455648, max timestamp=1731916455681Num uniques 859; 2024-11-18T07:54:15,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:15,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916515722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:15,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916515724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916515736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:15,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916515736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:15,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:15,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T07:54:15,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T07:54:15,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:15,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:15,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:15,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:15,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:15,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916515848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:15,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916515848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:15,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:15,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916515849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:15,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916515849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:15,897 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/e7ae50cf8eac4bb4bb731f91e22e9267 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/e7ae50cf8eac4bb4bb731f91e22e9267 2024-11-18T07:54:15,910 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/C of deeef93d343bf8d154ce5359480346cc into e7ae50cf8eac4bb4bb731f91e22e9267(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:15,910 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:15,910 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/C, priority=13, startTime=1731916455296; duration=0sec 2024-11-18T07:54:15,910 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:15,910 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:C 2024-11-18T07:54:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T07:54:15,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T07:54:15,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:15,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:15,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:15,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:15,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:16,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916516058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:16,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:16,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916516059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:16,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916516059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:16,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:16,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:16,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916516059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:16,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/fc8ee9c21a7c4261bf18f86fd1c904f2 2024-11-18T07:54:16,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/f8f7de3b42da40578b2c1c1dcaef6abc as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/f8f7de3b42da40578b2c1c1dcaef6abc 2024-11-18T07:54:16,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/f8f7de3b42da40578b2c1c1dcaef6abc, entries=200, sequenceid=379, filesize=14.4 K 2024-11-18T07:54:16,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/d1f9706d18fc4149a88b7734c90eac54 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/d1f9706d18fc4149a88b7734c90eac54 2024-11-18T07:54:16,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/d1f9706d18fc4149a88b7734c90eac54, entries=100, sequenceid=379, filesize=9.6 K 2024-11-18T07:54:16,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/fc8ee9c21a7c4261bf18f86fd1c904f2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/fc8ee9c21a7c4261bf18f86fd1c904f2 2024-11-18T07:54:16,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/fc8ee9c21a7c4261bf18f86fd1c904f2, entries=100, sequenceid=379, filesize=9.6 K 2024-11-18T07:54:16,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for deeef93d343bf8d154ce5359480346cc in 512ms, sequenceid=379, compaction requested=true 2024-11-18T07:54:16,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:16,117 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:16,118 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39919 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:16,118 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/A is initiating minor compaction (all files) 2024-11-18T07:54:16,118 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/A in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:16,118 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/d397777acf464079b025125fdbe283fe, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/336e7065bb0945f39dc3f53d551ce380, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/f8f7de3b42da40578b2c1c1dcaef6abc] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=39.0 K 2024-11-18T07:54:16,119 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d397777acf464079b025125fdbe283fe, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731916452151 2024-11-18T07:54:16,119 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 336e7065bb0945f39dc3f53d551ce380, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731916454399 2024-11-18T07:54:16,119 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8f7de3b42da40578b2c1c1dcaef6abc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731916455588 2024-11-18T07:54:16,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:16,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:16,123 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:16,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:16,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:16,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:16,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:16,124 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:16,124 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/B is initiating minor compaction (all files) 2024-11-18T07:54:16,124 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/B in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:16,124 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/948e60bf8faf4b2cb02ee0da37d1561b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/22bded27da7f4a908695956e2cbadac2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/d1f9706d18fc4149a88b7734c90eac54] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=34.2 K 2024-11-18T07:54:16,124 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 948e60bf8faf4b2cb02ee0da37d1561b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731916452151 2024-11-18T07:54:16,125 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 22bded27da7f4a908695956e2cbadac2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731916454399 2024-11-18T07:54:16,128 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting d1f9706d18fc4149a88b7734c90eac54, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731916455599 2024-11-18T07:54:16,137 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#A#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:16,137 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/614ff69c7c174ee1a243609eec1247ce is 50, key is test_row_0/A:col10/1731916455604/Put/seqid=0 2024-11-18T07:54:16,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T07:54:16,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:16,139 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-18T07:54:16,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:16,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:16,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:16,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:16,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:16,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:16,151 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#B#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:16,151 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/fd187852faba47e6b66e76fa90c9ab53 is 50, key is test_row_0/B:col10/1731916455604/Put/seqid=0 2024-11-18T07:54:16,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/2c078a7818cc4332b5cadac2bd9f7bf8 is 50, key is test_row_0/A:col10/1731916455734/Put/seqid=0 2024-11-18T07:54:16,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742327_1503 (size=12983) 2024-11-18T07:54:16,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742328_1504 (size=12983) 2024-11-18T07:54:16,218 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/614ff69c7c174ee1a243609eec1247ce as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/614ff69c7c174ee1a243609eec1247ce 2024-11-18T07:54:16,222 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/fd187852faba47e6b66e76fa90c9ab53 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fd187852faba47e6b66e76fa90c9ab53 2024-11-18T07:54:16,226 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/A of deeef93d343bf8d154ce5359480346cc into 614ff69c7c174ee1a243609eec1247ce(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:16,226 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:16,226 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/A, priority=13, startTime=1731916456117; duration=0sec 2024-11-18T07:54:16,226 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:16,226 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:A 2024-11-18T07:54:16,226 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:16,228 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:16,228 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/C is initiating minor compaction (all files) 2024-11-18T07:54:16,228 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/C in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:16,229 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/e7ae50cf8eac4bb4bb731f91e22e9267, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/dab5c79e44bb406c91baf5c42c543412, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/fc8ee9c21a7c4261bf18f86fd1c904f2] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=34.2 K 2024-11-18T07:54:16,229 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7ae50cf8eac4bb4bb731f91e22e9267, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731916452151 2024-11-18T07:54:16,229 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting dab5c79e44bb406c91baf5c42c543412, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731916454399 2024-11-18T07:54:16,230 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc8ee9c21a7c4261bf18f86fd1c904f2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731916455599 2024-11-18T07:54:16,230 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/B of deeef93d343bf8d154ce5359480346cc into fd187852faba47e6b66e76fa90c9ab53(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:16,230 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:16,230 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/B, priority=13, startTime=1731916456122; duration=0sec 2024-11-18T07:54:16,232 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:16,232 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:16,249 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#C#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:16,249 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/5374090b140b45fe8c23f9f24e2025a9 is 50, key is test_row_0/C:col10/1731916455604/Put/seqid=0 2024-11-18T07:54:16,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742329_1505 (size=12301) 2024-11-18T07:54:16,258 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/2c078a7818cc4332b5cadac2bd9f7bf8 2024-11-18T07:54:16,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742330_1506 (size=12983) 2024-11-18T07:54:16,287 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/5374090b140b45fe8c23f9f24e2025a9 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/5374090b140b45fe8c23f9f24e2025a9 2024-11-18T07:54:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T07:54:16,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/e29df486506a4053991d0315ea20311c is 50, key is test_row_0/B:col10/1731916455734/Put/seqid=0 2024-11-18T07:54:16,295 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/C of deeef93d343bf8d154ce5359480346cc into 5374090b140b45fe8c23f9f24e2025a9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:16,295 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:16,296 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/C, priority=13, startTime=1731916456123; duration=0sec 2024-11-18T07:54:16,296 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:16,296 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:C 2024-11-18T07:54:16,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742331_1507 (size=12301) 2024-11-18T07:54:16,338 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/e29df486506a4053991d0315ea20311c 2024-11-18T07:54:16,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/0e5d6e188e314683930eb35c1a157f35 is 50, key is test_row_0/C:col10/1731916455734/Put/seqid=0 2024-11-18T07:54:16,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742332_1508 (size=12301) 2024-11-18T07:54:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:16,383 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/0e5d6e188e314683930eb35c1a157f35 2024-11-18T07:54:16,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:16,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/2c078a7818cc4332b5cadac2bd9f7bf8 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/2c078a7818cc4332b5cadac2bd9f7bf8 2024-11-18T07:54:16,398 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/2c078a7818cc4332b5cadac2bd9f7bf8, entries=150, sequenceid=416, filesize=12.0 K 2024-11-18T07:54:16,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/e29df486506a4053991d0315ea20311c as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/e29df486506a4053991d0315ea20311c 2024-11-18T07:54:16,403 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/e29df486506a4053991d0315ea20311c, entries=150, sequenceid=416, filesize=12.0 K 2024-11-18T07:54:16,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/0e5d6e188e314683930eb35c1a157f35 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/0e5d6e188e314683930eb35c1a157f35 2024-11-18T07:54:16,408 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/0e5d6e188e314683930eb35c1a157f35, entries=150, sequenceid=416, filesize=12.0 K 2024-11-18T07:54:16,410 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=20.13 KB/20610 for deeef93d343bf8d154ce5359480346cc in 272ms, sequenceid=416, compaction requested=false 2024-11-18T07:54:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-18T07:54:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-18T07:54:16,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-18T07:54:16,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 746 msec 2024-11-18T07:54:16,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 755 msec 2024-11-18T07:54:16,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:54:16,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:16,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:16,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:16,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:16,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:16,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:16,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/c2a0ad91fb7649368df2ec66995cd13c is 50, key is test_row_0/A:col10/1731916456416/Put/seqid=0 2024-11-18T07:54:16,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742333_1509 (size=12301) 2024-11-18T07:54:16,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/c2a0ad91fb7649368df2ec66995cd13c 2024-11-18T07:54:16,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/ab5b76a83ed14ea58ecf8cc98128a705 is 50, key is test_row_0/B:col10/1731916456416/Put/seqid=0 2024-11-18T07:54:16,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742334_1510 (size=12301) 2024-11-18T07:54:16,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/ab5b76a83ed14ea58ecf8cc98128a705 2024-11-18T07:54:16,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/6b853bc2303245919c884e17a435aadf is 50, key is test_row_0/C:col10/1731916456416/Put/seqid=0 2024-11-18T07:54:16,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742335_1511 (size=12301) 2024-11-18T07:54:16,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/6b853bc2303245919c884e17a435aadf 2024-11-18T07:54:16,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/c2a0ad91fb7649368df2ec66995cd13c as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c2a0ad91fb7649368df2ec66995cd13c 2024-11-18T07:54:16,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c2a0ad91fb7649368df2ec66995cd13c, entries=150, sequenceid=429, filesize=12.0 K 2024-11-18T07:54:16,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/ab5b76a83ed14ea58ecf8cc98128a705 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/ab5b76a83ed14ea58ecf8cc98128a705 2024-11-18T07:54:16,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916456558, max timestamp=1731916456578Num uniques 859; 2024-11-18T07:54:16,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:16,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/ab5b76a83ed14ea58ecf8cc98128a705, entries=150, sequenceid=429, filesize=12.0 K 2024-11-18T07:54:16,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916456558, max timestamp=1731916456578Num uniques 859; 2024-11-18T07:54:16,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:16,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916456558, max timestamp=1731916456578Num uniques 859; 2024-11-18T07:54:16,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:16,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/6b853bc2303245919c884e17a435aadf as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6b853bc2303245919c884e17a435aadf 2024-11-18T07:54:16,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6b853bc2303245919c884e17a435aadf, entries=150, sequenceid=429, filesize=12.0 K 2024-11-18T07:54:16,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=46.96 KB/48090 for deeef93d343bf8d154ce5359480346cc in 179ms, sequenceid=429, compaction requested=true 2024-11-18T07:54:16,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:16,603 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:16,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:16,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:16,603 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:16,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:16,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:16,604 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/A is initiating minor compaction (all files) 2024-11-18T07:54:16,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:16,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:54:16,604 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/A in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:16,604 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/614ff69c7c174ee1a243609eec1247ce, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/2c078a7818cc4332b5cadac2bd9f7bf8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c2a0ad91fb7649368df2ec66995cd13c] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=36.7 K 2024-11-18T07:54:16,604 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:16,604 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 614ff69c7c174ee1a243609eec1247ce, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731916454415 2024-11-18T07:54:16,604 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c078a7818cc4332b5cadac2bd9f7bf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731916455720 2024-11-18T07:54:16,605 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2a0ad91fb7649368df2ec66995cd13c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731916456383 2024-11-18T07:54:16,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-18T07:54:16,605 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:16,605 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/B is initiating minor compaction (all files) 2024-11-18T07:54:16,605 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/B in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:16,605 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fd187852faba47e6b66e76fa90c9ab53, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/e29df486506a4053991d0315ea20311c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/ab5b76a83ed14ea58ecf8cc98128a705] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=36.7 K 2024-11-18T07:54:16,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:16,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:16,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:16,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:16,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:16,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:16,608 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting fd187852faba47e6b66e76fa90c9ab53, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731916454415 2024-11-18T07:54:16,609 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting e29df486506a4053991d0315ea20311c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731916455720 2024-11-18T07:54:16,609 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting ab5b76a83ed14ea58ecf8cc98128a705, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731916456383 2024-11-18T07:54:16,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/99b1ad6df3a0413caf111422e34cd310 is 50, key is test_row_0/A:col10/1731916456593/Put/seqid=0 2024-11-18T07:54:16,636 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#A#compaction#439 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:16,637 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/04afb62da37441ad8515605104b314a3 is 50, key is test_row_0/A:col10/1731916456416/Put/seqid=0 2024-11-18T07:54:16,645 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#B#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:16,645 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/d822e6fdbd544867af3b68cf279fdf47 is 50, key is test_row_0/B:col10/1731916456416/Put/seqid=0 2024-11-18T07:54:16,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742338_1514 (size=13085) 2024-11-18T07:54:16,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742336_1512 (size=17181) 2024-11-18T07:54:16,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742337_1513 (size=13085) 2024-11-18T07:54:16,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/99b1ad6df3a0413caf111422e34cd310 2024-11-18T07:54:16,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/92ce68c099dd4a02bf7d393271eec848 is 50, key is test_row_0/B:col10/1731916456593/Put/seqid=0 2024-11-18T07:54:16,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:16,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916516703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:16,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37826 deadline: 1731916516712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:16,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916456651, max timestamp=1731916456739Num uniques 859; 2024-11-18T07:54:16,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916456651, max timestamp=1731916456739Num uniques 859; 2024-11-18T07:54:16,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:16,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:16,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916456651, max timestamp=1731916456739Num uniques 859; 2024-11-18T07:54:16,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742339_1515 (size=12301) 2024-11-18T07:54:16,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:16,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/92ce68c099dd4a02bf7d393271eec848 2024-11-18T07:54:16,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/6fe753584a954e5181776e7c51b51e76 is 50, key is test_row_0/C:col10/1731916456593/Put/seqid=0 2024-11-18T07:54:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T07:54:16,806 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:16,806 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:54:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-18T07:54:16,809 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:54:16,810 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:54:16,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T07:54:16,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742340_1516 (size=12301) 2024-11-18T07:54:16,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/6fe753584a954e5181776e7c51b51e76 2024-11-18T07:54:16,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/99b1ad6df3a0413caf111422e34cd310 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/99b1ad6df3a0413caf111422e34cd310 2024-11-18T07:54:16,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/99b1ad6df3a0413caf111422e34cd310, entries=250, sequenceid=454, filesize=16.8 K 2024-11-18T07:54:16,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/92ce68c099dd4a02bf7d393271eec848 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/92ce68c099dd4a02bf7d393271eec848 2024-11-18T07:54:16,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/92ce68c099dd4a02bf7d393271eec848, entries=150, sequenceid=454, filesize=12.0 K 2024-11-18T07:54:16,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/6fe753584a954e5181776e7c51b51e76 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6fe753584a954e5181776e7c51b51e76 2024-11-18T07:54:16,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6fe753584a954e5181776e7c51b51e76, entries=150, sequenceid=454, filesize=12.0 K 2024-11-18T07:54:16,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=93.93 KB/96180 for deeef93d343bf8d154ce5359480346cc in 258ms, sequenceid=454, compaction requested=true 2024-11-18T07:54:16,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:16,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:A, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:16,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:54:16,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:B, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:16,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T07:54:16,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:16,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-18T07:54:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:16,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=100.63 KB heapSize=280.69 KB 2024-11-18T07:54:16,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:16,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:16,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:16,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:16,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:16,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:16,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/99692a0906924383a23bc00e83eed9b1 is 50, key is test_row_0/A:col10/1731916456870/Put/seqid=0 2024-11-18T07:54:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T07:54:16,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742341_1517 (size=14741) 2024-11-18T07:54:16,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/99692a0906924383a23bc00e83eed9b1 2024-11-18T07:54:16,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/745e71b6012949679478fc46341fd79a is 50, key is test_row_0/B:col10/1731916456870/Put/seqid=0 2024-11-18T07:54:16,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-18T07:54:16,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:16,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:16,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:16,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:16,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:16,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916516946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:16,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:16,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:16,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916516950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:16,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916516942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:16,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37826 deadline: 1731916516960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:16,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916516988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:17,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742342_1518 (size=12301) 2024-11-18T07:54:17,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/745e71b6012949679478fc46341fd79a 2024-11-18T07:54:17,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/37f86c9b9ae049a0a3ce3c4cb19994e5 is 50, key is test_row_0/C:col10/1731916456870/Put/seqid=0 2024-11-18T07:54:17,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742343_1519 (size=12301) 2024-11-18T07:54:17,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/37f86c9b9ae049a0a3ce3c4cb19994e5 2024-11-18T07:54:17,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/99692a0906924383a23bc00e83eed9b1 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/99692a0906924383a23bc00e83eed9b1 2024-11-18T07:54:17,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/99692a0906924383a23bc00e83eed9b1, entries=200, sequenceid=485, filesize=14.4 K 2024-11-18T07:54:17,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/745e71b6012949679478fc46341fd79a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/745e71b6012949679478fc46341fd79a 2024-11-18T07:54:17,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/745e71b6012949679478fc46341fd79a, entries=150, sequenceid=485, filesize=12.0 K 2024-11-18T07:54:17,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/37f86c9b9ae049a0a3ce3c4cb19994e5 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/37f86c9b9ae049a0a3ce3c4cb19994e5 2024-11-18T07:54:17,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/37f86c9b9ae049a0a3ce3c4cb19994e5, entries=150, sequenceid=485, filesize=12.0 K 2024-11-18T07:54:17,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~280.64 KB/287376, currentSize=80.51 KB/82440 for deeef93d343bf8d154ce5359480346cc in 198ms, sequenceid=485, compaction requested=true 2024-11-18T07:54:17,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:17,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:A, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:17,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2024-11-18T07:54:17,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:B, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:17,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=6), splitQueue=0 2024-11-18T07:54:17,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:17,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2024-11-18T07:54:17,098 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/d822e6fdbd544867af3b68cf279fdf47 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/d822e6fdbd544867af3b68cf279fdf47 2024-11-18T07:54:17,108 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/04afb62da37441ad8515605104b314a3 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/04afb62da37441ad8515605104b314a3 2024-11-18T07:54:17,113 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/B of deeef93d343bf8d154ce5359480346cc into d822e6fdbd544867af3b68cf279fdf47(size=12.8 K), total size for store is 36.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:17,113 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:17,113 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/B, priority=13, startTime=1731916456603; duration=0sec 2024-11-18T07:54:17,114 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2024-11-18T07:54:17,114 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:17,114 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-18T07:54:17,119 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T07:54:17,119 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T07:54:17,119 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. because compaction request was cancelled 2024-11-18T07:54:17,119 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:A 2024-11-18T07:54:17,119 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-18T07:54:17,120 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/A of deeef93d343bf8d154ce5359480346cc into 04afb62da37441ad8515605104b314a3(size=12.8 K), total size for store is 44.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:17,120 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:17,120 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/A, priority=13, startTime=1731916456603; duration=0sec 2024-11-18T07:54:17,120 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2024-11-18T07:54:17,120 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:A 2024-11-18T07:54:17,120 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:17,121 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62187 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-18T07:54:17,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-18T07:54:17,122 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/C is initiating minor compaction (all files) 2024-11-18T07:54:17,122 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/C in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:17,122 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/5374090b140b45fe8c23f9f24e2025a9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/0e5d6e188e314683930eb35c1a157f35, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6b853bc2303245919c884e17a435aadf, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6fe753584a954e5181776e7c51b51e76, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/37f86c9b9ae049a0a3ce3c4cb19994e5] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=60.7 K 2024-11-18T07:54:17,122 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T07:54:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:17,122 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5374090b140b45fe8c23f9f24e2025a9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731916454415 2024-11-18T07:54:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:17,123 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e5d6e188e314683930eb35c1a157f35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731916455720 2024-11-18T07:54:17,123 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b853bc2303245919c884e17a435aadf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731916456383 2024-11-18T07:54:17,123 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fe753584a954e5181776e7c51b51e76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731916456585 2024-11-18T07:54:17,124 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 37f86c9b9ae049a0a3ce3c4cb19994e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731916456821 2024-11-18T07:54:17,125 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-18T07:54:17,125 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T07:54:17,125 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T07:54:17,125 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. because compaction request was cancelled 2024-11-18T07:54:17,125 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:C 2024-11-18T07:54:17,125 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:A 2024-11-18T07:54:17,125 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:17,126 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:17,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/58045fb6994b4a7f903953415d7c9efa is 50, key is test_row_0/A:col10/1731916456984/Put/seqid=0 2024-11-18T07:54:17,128 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45007 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:17,128 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/A is initiating minor compaction (all files) 2024-11-18T07:54:17,128 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/A in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:17,129 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/04afb62da37441ad8515605104b314a3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/99b1ad6df3a0413caf111422e34cd310, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/99692a0906924383a23bc00e83eed9b1] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=44.0 K 2024-11-18T07:54:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T07:54:17,130 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04afb62da37441ad8515605104b314a3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731916456383 2024-11-18T07:54:17,130 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99b1ad6df3a0413caf111422e34cd310, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731916456564 2024-11-18T07:54:17,131 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99692a0906924383a23bc00e83eed9b1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731916456821 2024-11-18T07:54:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:17,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. as already flushing 2024-11-18T07:54:17,144 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#C#compaction#447 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:17,144 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/f726e9a5505f4b4ea28a932c5e4b9967 is 50, key is test_row_0/C:col10/1731916456870/Put/seqid=0 2024-11-18T07:54:17,152 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#A#compaction#448 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:17,153 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/fe2d9879eb5743a8b0e467f91d6b33d0 is 50, key is test_row_0/A:col10/1731916456870/Put/seqid=0 2024-11-18T07:54:17,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742344_1520 (size=12301) 2024-11-18T07:54:17,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742345_1521 (size=13153) 2024-11-18T07:54:17,167 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/58045fb6994b4a7f903953415d7c9efa 2024-11-18T07:54:17,173 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/f726e9a5505f4b4ea28a932c5e4b9967 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/f726e9a5505f4b4ea28a932c5e4b9967 2024-11-18T07:54:17,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742346_1522 (size=13187) 2024-11-18T07:54:17,185 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in deeef93d343bf8d154ce5359480346cc/C of deeef93d343bf8d154ce5359480346cc into f726e9a5505f4b4ea28a932c5e4b9967(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:17,186 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:17,186 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/C, priority=11, startTime=1731916457086; duration=0sec 2024-11-18T07:54:17,186 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T07:54:17,186 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:C 2024-11-18T07:54:17,186 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:17,188 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:17,188 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): deeef93d343bf8d154ce5359480346cc/B is initiating minor compaction (all files) 2024-11-18T07:54:17,188 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of deeef93d343bf8d154ce5359480346cc/B in TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:17,189 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/d822e6fdbd544867af3b68cf279fdf47, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/92ce68c099dd4a02bf7d393271eec848, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/745e71b6012949679478fc46341fd79a] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp, totalSize=36.8 K 2024-11-18T07:54:17,189 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting d822e6fdbd544867af3b68cf279fdf47, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731916456383 2024-11-18T07:54:17,190 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 92ce68c099dd4a02bf7d393271eec848, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731916456585 2024-11-18T07:54:17,190 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 745e71b6012949679478fc46341fd79a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731916456821 2024-11-18T07:54:17,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/09e36ac4e4c44f29b70ee9c96fe425c8 is 50, key is test_row_0/B:col10/1731916456984/Put/seqid=0 2024-11-18T07:54:17,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916517179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:17,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742347_1523 (size=12301) 2024-11-18T07:54:17,211 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): deeef93d343bf8d154ce5359480346cc#B#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:17,211 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/f19893d7d5314b4983e8a4770cba6b67 is 50, key is test_row_0/B:col10/1731916456870/Put/seqid=0 2024-11-18T07:54:17,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742348_1524 (size=13187) 2024-11-18T07:54:17,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916517193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:17,221 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/f19893d7d5314b4983e8a4770cba6b67 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/f19893d7d5314b4983e8a4770cba6b67 2024-11-18T07:54:17,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916517200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:17,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916517202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:17,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37826 deadline: 1731916517204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:17,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:17,231 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/B of deeef93d343bf8d154ce5359480346cc into f19893d7d5314b4983e8a4770cba6b67(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:17,231 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:17,231 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/B, priority=13, startTime=1731916456862; duration=0sec 2024-11-18T07:54:17,231 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:54:17,231 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:17,231 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:C 2024-11-18T07:54:17,231 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:17,231 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-18T07:54:17,232 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T07:54:17,232 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T07:54:17,233 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. because compaction request was cancelled 2024-11-18T07:54:17,233 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:C 2024-11-18T07:54:17,233 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-18T07:54:17,234 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T07:54:17,234 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T07:54:17,234 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. because compaction request was cancelled 2024-11-18T07:54:17,234 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:17,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916517308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:17,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:17,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916517329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:17,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37826 deadline: 1731916517329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:17,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916517329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:17,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916517329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:17,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:17,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:17,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:17,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T07:54:17,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:17,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1731916517528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:17,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:17,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37826 deadline: 1731916517549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:17,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37880 deadline: 1731916517550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:17,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37912 deadline: 1731916517550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:17,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:17,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37896 deadline: 1731916517550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:17,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., hostname=87fb3d252950,37987,1731916323128, seqNum=2 because the exception is null or not the one we care about 2024-11-18T07:54:17,580 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/fe2d9879eb5743a8b0e467f91d6b33d0 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/fe2d9879eb5743a8b0e467f91d6b33d0 2024-11-18T07:54:17,587 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in deeef93d343bf8d154ce5359480346cc/A of deeef93d343bf8d154ce5359480346cc into fe2d9879eb5743a8b0e467f91d6b33d0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:17,587 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:17,587 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., storeName=deeef93d343bf8d154ce5359480346cc/A, priority=13, startTime=1731916456862; duration=0sec 2024-11-18T07:54:17,587 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:17,587 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:A 2024-11-18T07:54:17,611 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/09e36ac4e4c44f29b70ee9c96fe425c8 2024-11-18T07:54:17,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/24ccdc63b6634aefb0daeffffcaba58e is 50, key is test_row_0/C:col10/1731916456984/Put/seqid=0 2024-11-18T07:54:17,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742349_1525 (size=12301) 2024-11-18T07:54:17,639 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/24ccdc63b6634aefb0daeffffcaba58e 2024-11-18T07:54:17,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/58045fb6994b4a7f903953415d7c9efa as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/58045fb6994b4a7f903953415d7c9efa 2024-11-18T07:54:17,649 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/58045fb6994b4a7f903953415d7c9efa, entries=150, sequenceid=502, filesize=12.0 K 2024-11-18T07:54:17,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/09e36ac4e4c44f29b70ee9c96fe425c8 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/09e36ac4e4c44f29b70ee9c96fe425c8 2024-11-18T07:54:17,658 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/09e36ac4e4c44f29b70ee9c96fe425c8, entries=150, sequenceid=502, filesize=12.0 K 2024-11-18T07:54:17,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/24ccdc63b6634aefb0daeffffcaba58e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/24ccdc63b6634aefb0daeffffcaba58e 2024-11-18T07:54:17,665 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/24ccdc63b6634aefb0daeffffcaba58e, entries=150, sequenceid=502, filesize=12.0 K 2024-11-18T07:54:17,666 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for deeef93d343bf8d154ce5359480346cc in 544ms, sequenceid=502, compaction requested=false 2024-11-18T07:54:17,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:17,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:17,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-18T07:54:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-18T07:54:17,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-18T07:54:17,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 857 msec 2024-11-18T07:54:17,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 864 msec 2024-11-18T07:54:17,830 INFO [Thread-1890 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1890. 2024-11-18T07:54:17,830 DEBUG [Thread-1890 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:17,830 DEBUG [Thread-1890 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,830 DEBUG [Thread-1890 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,830 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:17,834 INFO [Thread-1892 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1892. 2024-11-18T07:54:17,835 DEBUG [Thread-1892 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:17,835 DEBUG [Thread-1892 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,835 DEBUG [Thread-1892 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:17,840 INFO [Thread-1894 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1894. 2024-11-18T07:54:17,840 DEBUG [Thread-1894 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:17,840 DEBUG [Thread-1894 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,840 DEBUG [Thread-1894 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,840 INFO [Thread-1888 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1888. 2024-11-18T07:54:17,840 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:17,840 DEBUG [Thread-1888 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:17,840 DEBUG [Thread-1888 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,840 DEBUG [Thread-1888 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,840 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:17,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T07:54:17,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:17,843 INFO [Thread-1875 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1875. 2024-11-18T07:54:17,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:17,843 DEBUG [Thread-1875 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:17,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:17,843 DEBUG [Thread-1875 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:17,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:17,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:17,843 DEBUG [Thread-1875 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,843 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:17,848 INFO [Thread-1886 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1886. 2024-11-18T07:54:17,848 DEBUG [Thread-1886 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:17,848 DEBUG [Thread-1886 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,849 DEBUG [Thread-1886 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:17,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/2af91fd273c04077983e2a6f5ee75d94 is 50, key is test_row_0/A:col10/1731916457199/Put/seqid=0 2024-11-18T07:54:17,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742350_1526 (size=14741) 2024-11-18T07:54:17,873 INFO [Thread-1881 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1881. 2024-11-18T07:54:17,873 DEBUG [Thread-1881 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:17,873 DEBUG [Thread-1881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,873 DEBUG [Thread-1881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:17,875 INFO [Thread-1883 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1883. 2024-11-18T07:54:17,875 DEBUG [Thread-1883 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:17,875 DEBUG [Thread-1883 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,875 DEBUG [Thread-1883 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,875 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:17,876 INFO [Thread-1879 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1879. 2024-11-18T07:54:17,876 DEBUG [Thread-1879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:17,876 DEBUG [Thread-1879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,876 DEBUG [Thread-1879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:17,879 INFO [Thread-1877 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1877. 2024-11-18T07:54:17,879 DEBUG [Thread-1877 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:17,879 DEBUG [Thread-1877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,879 DEBUG [Thread-1877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T07:54:17,949 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:17,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T07:54:17,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-18T07:54:17,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-18T07:54:17,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-11-18T07:54:17,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-18T07:54:17,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-18T07:54:17,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T07:54:17,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T07:54:17,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 822 2024-11-18T07:54:17,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2460 rows 2024-11-18T07:54:17,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 830 2024-11-18T07:54:17,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2470 rows 2024-11-18T07:54:17,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 823 2024-11-18T07:54:17,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2462 rows 2024-11-18T07:54:17,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 817 2024-11-18T07:54:17,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2449 rows 2024-11-18T07:54:17,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 815 2024-11-18T07:54:17,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2445 rows 2024-11-18T07:54:17,950 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T07:54:17,950 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T07:54:17,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:17,951 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T07:54:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T07:54:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T07:54:17,954 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916457953"}]},"ts":"1731916457953"} 2024-11-18T07:54:17,954 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:17,955 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T07:54:17,956 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T07:54:17,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T07:54:17,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=deeef93d343bf8d154ce5359480346cc, UNASSIGN}] 2024-11-18T07:54:17,958 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=deeef93d343bf8d154ce5359480346cc, UNASSIGN 2024-11-18T07:54:17,959 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=deeef93d343bf8d154ce5359480346cc, regionState=CLOSING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:54:17,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=deeef93d343bf8d154ce5359480346cc, UNASSIGN because future has completed 2024-11-18T07:54:17,960 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T07:54:17,961 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:54:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T07:54:18,113 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:18,113 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T07:54:18,113 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing deeef93d343bf8d154ce5359480346cc, disabling compactions & flushes 2024-11-18T07:54:18,113 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:18,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/2af91fd273c04077983e2a6f5ee75d94 2024-11-18T07:54:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T07:54:18,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/e8816c609b4d429a8769d7cf26342fcc is 50, key is test_row_0/B:col10/1731916457199/Put/seqid=0 2024-11-18T07:54:18,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742351_1527 (size=12301) 2024-11-18T07:54:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T07:54:18,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/e8816c609b4d429a8769d7cf26342fcc 2024-11-18T07:54:18,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/9d084d5efc414135912a5e5b38951b72 is 50, key is test_row_0/C:col10/1731916457199/Put/seqid=0 2024-11-18T07:54:18,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742352_1528 (size=12301) 2024-11-18T07:54:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T07:54:19,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/9d084d5efc414135912a5e5b38951b72 2024-11-18T07:54:19,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/2af91fd273c04077983e2a6f5ee75d94 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/2af91fd273c04077983e2a6f5ee75d94 2024-11-18T07:54:19,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/2af91fd273c04077983e2a6f5ee75d94, entries=200, sequenceid=524, filesize=14.4 K 2024-11-18T07:54:19,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/e8816c609b4d429a8769d7cf26342fcc as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/e8816c609b4d429a8769d7cf26342fcc 2024-11-18T07:54:19,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/e8816c609b4d429a8769d7cf26342fcc, entries=150, sequenceid=524, filesize=12.0 K 2024-11-18T07:54:19,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/9d084d5efc414135912a5e5b38951b72 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/9d084d5efc414135912a5e5b38951b72 2024-11-18T07:54:19,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/9d084d5efc414135912a5e5b38951b72, entries=150, sequenceid=524, filesize=12.0 K 2024-11-18T07:54:19,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=26.84 KB/27480 for deeef93d343bf8d154ce5359480346cc in 1268ms, sequenceid=524, compaction requested=true 2024-11-18T07:54:19,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for deeef93d343bf8d154ce5359480346cc: 2024-11-18T07:54:19,110 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:19,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:19,110 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:19,110 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. because compaction request was cancelled 2024-11-18T07:54:19,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:19,110 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:A 2024-11-18T07:54:19,110 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. after waiting 0 ms 2024-11-18T07:54:19,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:19,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:19,110 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:19,110 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. because compaction request was cancelled 2024-11-18T07:54:19,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store deeef93d343bf8d154ce5359480346cc:C, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:19,110 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:B 2024-11-18T07:54:19,110 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. because compaction request was cancelled 2024-11-18T07:54:19,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:19,110 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: deeef93d343bf8d154ce5359480346cc:C 2024-11-18T07:54:19,110 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing deeef93d343bf8d154ce5359480346cc 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-18T07:54:19,111 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=A 2024-11-18T07:54:19,111 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:19,111 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=B 2024-11-18T07:54:19,111 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:19,111 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK deeef93d343bf8d154ce5359480346cc, store=C 2024-11-18T07:54:19,111 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:19,115 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/f250916cb6914473a392bd7bb5545e1b is 50, key is test_row_1/A:col10/1731916457869/Put/seqid=0 2024-11-18T07:54:19,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742353_1529 (size=9857) 2024-11-18T07:54:19,519 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/f250916cb6914473a392bd7bb5545e1b 2024-11-18T07:54:19,524 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/82e02e942bab4b65b8ac7768bd9cfaaf is 50, key is test_row_1/B:col10/1731916457869/Put/seqid=0 2024-11-18T07:54:19,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742354_1530 (size=9857) 2024-11-18T07:54:19,928 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/82e02e942bab4b65b8ac7768bd9cfaaf 2024-11-18T07:54:19,933 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/85c6890d1db042de8c9c3567040f7c9c is 50, key is test_row_1/C:col10/1731916457869/Put/seqid=0 2024-11-18T07:54:19,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742355_1531 (size=9857) 2024-11-18T07:54:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T07:54:20,337 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/85c6890d1db042de8c9c3567040f7c9c 2024-11-18T07:54:20,341 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/A/f250916cb6914473a392bd7bb5545e1b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/f250916cb6914473a392bd7bb5545e1b 2024-11-18T07:54:20,351 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/f250916cb6914473a392bd7bb5545e1b, entries=100, sequenceid=531, filesize=9.6 K 2024-11-18T07:54:20,352 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/B/82e02e942bab4b65b8ac7768bd9cfaaf as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/82e02e942bab4b65b8ac7768bd9cfaaf 2024-11-18T07:54:20,356 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/82e02e942bab4b65b8ac7768bd9cfaaf, entries=100, sequenceid=531, filesize=9.6 K 2024-11-18T07:54:20,357 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/.tmp/C/85c6890d1db042de8c9c3567040f7c9c as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/85c6890d1db042de8c9c3567040f7c9c 2024-11-18T07:54:20,362 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/85c6890d1db042de8c9c3567040f7c9c, entries=100, sequenceid=531, filesize=9.6 K 2024-11-18T07:54:20,363 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for deeef93d343bf8d154ce5359480346cc in 1253ms, sequenceid=531, compaction requested=true 2024-11-18T07:54:20,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1b3762441d1e45c3979c666c9574d227, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/11fd46ef45224fa8b925b8dbbcbea65a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1480d781e694431981339386192cf5c9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/ba7522c7aafb42739561bd2e62c46db3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1c8b1dccba004854a8f0b83339933684, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/5c1b5f639d5848e2ab1168733ae29901, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c20880e01a67440fbe20f7902727e4dc, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/951d73d9ede64750829dd7824ebcd16f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/4465e0e5f5c042fcb31a9724a838c3d2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c60a75fa49c741a696c91b5a268d451b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/6d6a57c5773d4f3c87a85b33d9b50c19, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/fcf4f3b4160642028827f4e5d18254ab, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1ac62f6ccad24e09b856358b0ddbbfc4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/0dbfde4c811247c0a5576ab4848619cb, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c4df6961ffa24226bf967d0545e34c31, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/825f39eb9a454b82a498d4c85c3d1f02, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/07f12463ada541cfac2f50a02064c083, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/d397777acf464079b025125fdbe283fe, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/336e7065bb0945f39dc3f53d551ce380, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/f8f7de3b42da40578b2c1c1dcaef6abc, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/614ff69c7c174ee1a243609eec1247ce, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/2c078a7818cc4332b5cadac2bd9f7bf8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/04afb62da37441ad8515605104b314a3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c2a0ad91fb7649368df2ec66995cd13c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/99b1ad6df3a0413caf111422e34cd310, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/99692a0906924383a23bc00e83eed9b1] to archive 2024-11-18T07:54:20,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:54:20,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1b3762441d1e45c3979c666c9574d227 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1b3762441d1e45c3979c666c9574d227 2024-11-18T07:54:20,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/11fd46ef45224fa8b925b8dbbcbea65a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/11fd46ef45224fa8b925b8dbbcbea65a 2024-11-18T07:54:20,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1480d781e694431981339386192cf5c9 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1480d781e694431981339386192cf5c9 2024-11-18T07:54:20,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/ba7522c7aafb42739561bd2e62c46db3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/ba7522c7aafb42739561bd2e62c46db3 2024-11-18T07:54:20,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1c8b1dccba004854a8f0b83339933684 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1c8b1dccba004854a8f0b83339933684 2024-11-18T07:54:20,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/5c1b5f639d5848e2ab1168733ae29901 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/5c1b5f639d5848e2ab1168733ae29901 2024-11-18T07:54:20,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c20880e01a67440fbe20f7902727e4dc to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c20880e01a67440fbe20f7902727e4dc 2024-11-18T07:54:20,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/951d73d9ede64750829dd7824ebcd16f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/951d73d9ede64750829dd7824ebcd16f 2024-11-18T07:54:20,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/4465e0e5f5c042fcb31a9724a838c3d2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/4465e0e5f5c042fcb31a9724a838c3d2 2024-11-18T07:54:20,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c60a75fa49c741a696c91b5a268d451b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c60a75fa49c741a696c91b5a268d451b 2024-11-18T07:54:20,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/6d6a57c5773d4f3c87a85b33d9b50c19 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/6d6a57c5773d4f3c87a85b33d9b50c19 2024-11-18T07:54:20,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/fcf4f3b4160642028827f4e5d18254ab to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/fcf4f3b4160642028827f4e5d18254ab 2024-11-18T07:54:20,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1ac62f6ccad24e09b856358b0ddbbfc4 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/1ac62f6ccad24e09b856358b0ddbbfc4 2024-11-18T07:54:20,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/0dbfde4c811247c0a5576ab4848619cb to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/0dbfde4c811247c0a5576ab4848619cb 2024-11-18T07:54:20,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c4df6961ffa24226bf967d0545e34c31 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c4df6961ffa24226bf967d0545e34c31 2024-11-18T07:54:20,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/825f39eb9a454b82a498d4c85c3d1f02 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/825f39eb9a454b82a498d4c85c3d1f02 2024-11-18T07:54:20,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/07f12463ada541cfac2f50a02064c083 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/07f12463ada541cfac2f50a02064c083 2024-11-18T07:54:20,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/d397777acf464079b025125fdbe283fe to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/d397777acf464079b025125fdbe283fe 2024-11-18T07:54:20,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/336e7065bb0945f39dc3f53d551ce380 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/336e7065bb0945f39dc3f53d551ce380 2024-11-18T07:54:20,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/f8f7de3b42da40578b2c1c1dcaef6abc to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/f8f7de3b42da40578b2c1c1dcaef6abc 2024-11-18T07:54:20,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/614ff69c7c174ee1a243609eec1247ce to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/614ff69c7c174ee1a243609eec1247ce 2024-11-18T07:54:20,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/2c078a7818cc4332b5cadac2bd9f7bf8 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/2c078a7818cc4332b5cadac2bd9f7bf8 2024-11-18T07:54:20,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/04afb62da37441ad8515605104b314a3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/04afb62da37441ad8515605104b314a3 2024-11-18T07:54:20,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c2a0ad91fb7649368df2ec66995cd13c to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/c2a0ad91fb7649368df2ec66995cd13c 2024-11-18T07:54:20,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/99b1ad6df3a0413caf111422e34cd310 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/99b1ad6df3a0413caf111422e34cd310 2024-11-18T07:54:20,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/99692a0906924383a23bc00e83eed9b1 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/99692a0906924383a23bc00e83eed9b1 2024-11-18T07:54:20,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fa4a4463b78e46b8a72448ff56122d74, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/4347c206ad944cf0b7ae4414ed6c04ea, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2c6038cbcdc84db3ad9522613f7f764f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/530516bcd0524a319615739e1aa0c41f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/577085ad0d8b43dcb789b5db876521e3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/54a7da08a527410cb5fbab4ab664f1ee, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/7d00eaed894246bf940988256f1ef518, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/f9dd8da4f8bf4eb89ccff3f6c9299422, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/8466a9553f1d4a85bbfb99cb2fdb5532, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fabd0bf0185048b5a3f54225550bdd9d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/778e16f64eba4864906db41e9de2e297, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2ada0d0398c14d0283225370f349d331, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/8d9b4311c6ca44d79017b31a97a14a0f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/9eaba1dc858943f28211e6c205518f0b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2b217f65308e4925be92767caa50ca32, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/4c984d6c28dc4b6bbf8e0522c93ed0f5, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/948e60bf8faf4b2cb02ee0da37d1561b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fbabb5b09aac420e8ca4b967bef90f3e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/22bded27da7f4a908695956e2cbadac2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fd187852faba47e6b66e76fa90c9ab53, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/d1f9706d18fc4149a88b7734c90eac54, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/e29df486506a4053991d0315ea20311c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/d822e6fdbd544867af3b68cf279fdf47, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/ab5b76a83ed14ea58ecf8cc98128a705, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/92ce68c099dd4a02bf7d393271eec848, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/745e71b6012949679478fc46341fd79a] to archive 2024-11-18T07:54:20,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:54:20,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fa4a4463b78e46b8a72448ff56122d74 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fa4a4463b78e46b8a72448ff56122d74 2024-11-18T07:54:20,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/4347c206ad944cf0b7ae4414ed6c04ea to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/4347c206ad944cf0b7ae4414ed6c04ea 2024-11-18T07:54:20,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2c6038cbcdc84db3ad9522613f7f764f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2c6038cbcdc84db3ad9522613f7f764f 2024-11-18T07:54:20,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/530516bcd0524a319615739e1aa0c41f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/530516bcd0524a319615739e1aa0c41f 2024-11-18T07:54:20,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/577085ad0d8b43dcb789b5db876521e3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/577085ad0d8b43dcb789b5db876521e3 2024-11-18T07:54:20,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/54a7da08a527410cb5fbab4ab664f1ee to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/54a7da08a527410cb5fbab4ab664f1ee 2024-11-18T07:54:20,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/7d00eaed894246bf940988256f1ef518 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/7d00eaed894246bf940988256f1ef518 2024-11-18T07:54:20,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/f9dd8da4f8bf4eb89ccff3f6c9299422 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/f9dd8da4f8bf4eb89ccff3f6c9299422 2024-11-18T07:54:20,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/8466a9553f1d4a85bbfb99cb2fdb5532 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/8466a9553f1d4a85bbfb99cb2fdb5532 2024-11-18T07:54:20,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fabd0bf0185048b5a3f54225550bdd9d to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fabd0bf0185048b5a3f54225550bdd9d 2024-11-18T07:54:20,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/778e16f64eba4864906db41e9de2e297 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/778e16f64eba4864906db41e9de2e297 2024-11-18T07:54:20,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2ada0d0398c14d0283225370f349d331 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2ada0d0398c14d0283225370f349d331 2024-11-18T07:54:20,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/8d9b4311c6ca44d79017b31a97a14a0f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/8d9b4311c6ca44d79017b31a97a14a0f 2024-11-18T07:54:20,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/9eaba1dc858943f28211e6c205518f0b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/9eaba1dc858943f28211e6c205518f0b 2024-11-18T07:54:20,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2b217f65308e4925be92767caa50ca32 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/2b217f65308e4925be92767caa50ca32 2024-11-18T07:54:20,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/4c984d6c28dc4b6bbf8e0522c93ed0f5 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/4c984d6c28dc4b6bbf8e0522c93ed0f5 2024-11-18T07:54:20,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/948e60bf8faf4b2cb02ee0da37d1561b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/948e60bf8faf4b2cb02ee0da37d1561b 2024-11-18T07:54:20,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fbabb5b09aac420e8ca4b967bef90f3e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fbabb5b09aac420e8ca4b967bef90f3e 2024-11-18T07:54:20,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/22bded27da7f4a908695956e2cbadac2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/22bded27da7f4a908695956e2cbadac2 2024-11-18T07:54:20,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fd187852faba47e6b66e76fa90c9ab53 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/fd187852faba47e6b66e76fa90c9ab53 2024-11-18T07:54:20,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/d1f9706d18fc4149a88b7734c90eac54 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/d1f9706d18fc4149a88b7734c90eac54 2024-11-18T07:54:20,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/e29df486506a4053991d0315ea20311c to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/e29df486506a4053991d0315ea20311c 2024-11-18T07:54:20,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/d822e6fdbd544867af3b68cf279fdf47 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/d822e6fdbd544867af3b68cf279fdf47 2024-11-18T07:54:20,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/ab5b76a83ed14ea58ecf8cc98128a705 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/ab5b76a83ed14ea58ecf8cc98128a705 2024-11-18T07:54:20,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/92ce68c099dd4a02bf7d393271eec848 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/92ce68c099dd4a02bf7d393271eec848 2024-11-18T07:54:20,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/745e71b6012949679478fc46341fd79a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/745e71b6012949679478fc46341fd79a 2024-11-18T07:54:20,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/e974f635388d4402a3a6dd9fabfacfc9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/82cd47cd819a48a2891e1f556337c824, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/4679df8767554b9592723a25d74423b4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/261d30c401564939b23fa7b7f72197fa, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/f825e5e5b3e94846a4ef0fcbcf5aa608, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/b057635b4942436aa3898121c4ecd213, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/aa8d3ec975c74924a04d3ff119ea1d69, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/cc9afd305d7848829d473e6f222807af, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d3ffc0d780b04221ac9933680f02fd11, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/494a8b801b2d473e988e4e2c888c3fd3, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/392106e24b5b40349d40abd8acce5d99, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/27b033c1914246aabacae8dfd4fc5cf4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/c1ab4544b9fd4b08941e41b1732800e9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/fb83198d6ac0402b94bd19384f8bdc94, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/12e6664c4bb44a5e95270d6b639c4da2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/8640b5df872e4616b52f469cc978c0db, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/e7ae50cf8eac4bb4bb731f91e22e9267, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d12b3b8eb5c34ca7b819097c56bf74a5, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/dab5c79e44bb406c91baf5c42c543412, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/5374090b140b45fe8c23f9f24e2025a9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/fc8ee9c21a7c4261bf18f86fd1c904f2, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/0e5d6e188e314683930eb35c1a157f35, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6b853bc2303245919c884e17a435aadf, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6fe753584a954e5181776e7c51b51e76, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/37f86c9b9ae049a0a3ce3c4cb19994e5] to archive 2024-11-18T07:54:20,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:54:20,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/e974f635388d4402a3a6dd9fabfacfc9 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/e974f635388d4402a3a6dd9fabfacfc9 2024-11-18T07:54:20,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/82cd47cd819a48a2891e1f556337c824 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/82cd47cd819a48a2891e1f556337c824 2024-11-18T07:54:20,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/4679df8767554b9592723a25d74423b4 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/4679df8767554b9592723a25d74423b4 2024-11-18T07:54:20,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/261d30c401564939b23fa7b7f72197fa to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/261d30c401564939b23fa7b7f72197fa 2024-11-18T07:54:20,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/f825e5e5b3e94846a4ef0fcbcf5aa608 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/f825e5e5b3e94846a4ef0fcbcf5aa608 2024-11-18T07:54:20,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/b057635b4942436aa3898121c4ecd213 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/b057635b4942436aa3898121c4ecd213 2024-11-18T07:54:20,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/aa8d3ec975c74924a04d3ff119ea1d69 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/aa8d3ec975c74924a04d3ff119ea1d69 2024-11-18T07:54:20,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/cc9afd305d7848829d473e6f222807af to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/cc9afd305d7848829d473e6f222807af 2024-11-18T07:54:20,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d3ffc0d780b04221ac9933680f02fd11 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d3ffc0d780b04221ac9933680f02fd11 2024-11-18T07:54:20,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/494a8b801b2d473e988e4e2c888c3fd3 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/494a8b801b2d473e988e4e2c888c3fd3 2024-11-18T07:54:20,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/392106e24b5b40349d40abd8acce5d99 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/392106e24b5b40349d40abd8acce5d99 2024-11-18T07:54:20,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/27b033c1914246aabacae8dfd4fc5cf4 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/27b033c1914246aabacae8dfd4fc5cf4 2024-11-18T07:54:20,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/c1ab4544b9fd4b08941e41b1732800e9 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/c1ab4544b9fd4b08941e41b1732800e9 2024-11-18T07:54:20,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/fb83198d6ac0402b94bd19384f8bdc94 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/fb83198d6ac0402b94bd19384f8bdc94 2024-11-18T07:54:20,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/12e6664c4bb44a5e95270d6b639c4da2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/12e6664c4bb44a5e95270d6b639c4da2 2024-11-18T07:54:20,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/8640b5df872e4616b52f469cc978c0db to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/8640b5df872e4616b52f469cc978c0db 2024-11-18T07:54:20,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/e7ae50cf8eac4bb4bb731f91e22e9267 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/e7ae50cf8eac4bb4bb731f91e22e9267 2024-11-18T07:54:20,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d12b3b8eb5c34ca7b819097c56bf74a5 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/d12b3b8eb5c34ca7b819097c56bf74a5 2024-11-18T07:54:20,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/dab5c79e44bb406c91baf5c42c543412 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/dab5c79e44bb406c91baf5c42c543412 2024-11-18T07:54:20,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/5374090b140b45fe8c23f9f24e2025a9 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/5374090b140b45fe8c23f9f24e2025a9 2024-11-18T07:54:20,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/fc8ee9c21a7c4261bf18f86fd1c904f2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/fc8ee9c21a7c4261bf18f86fd1c904f2 2024-11-18T07:54:20,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/0e5d6e188e314683930eb35c1a157f35 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/0e5d6e188e314683930eb35c1a157f35 2024-11-18T07:54:20,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6b853bc2303245919c884e17a435aadf to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6b853bc2303245919c884e17a435aadf 2024-11-18T07:54:20,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6fe753584a954e5181776e7c51b51e76 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/6fe753584a954e5181776e7c51b51e76 2024-11-18T07:54:20,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/37f86c9b9ae049a0a3ce3c4cb19994e5 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/37f86c9b9ae049a0a3ce3c4cb19994e5 2024-11-18T07:54:20,436 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/recovered.edits/534.seqid, newMaxSeqId=534, maxSeqId=1 2024-11-18T07:54:20,436 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. 2024-11-18T07:54:20,437 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for deeef93d343bf8d154ce5359480346cc: Waiting for close lock at 1731916458113Running coprocessor pre-close hooks at 1731916458113Disabling compacts and flushes for region at 1731916458113Disabling writes for close at 1731916459110 (+997 ms)Obtaining lock to block concurrent updates at 1731916459110Preparing flush snapshotting stores in deeef93d343bf8d154ce5359480346cc at 1731916459110Finished memstore snapshotting TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731916459111 (+1 ms)Flushing stores of TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc. at 1731916459111Flushing deeef93d343bf8d154ce5359480346cc/A: creating writer at 1731916459111Flushing deeef93d343bf8d154ce5359480346cc/A: appending metadata at 1731916459114 (+3 ms)Flushing deeef93d343bf8d154ce5359480346cc/A: closing flushed file at 1731916459114Flushing deeef93d343bf8d154ce5359480346cc/B: creating writer at 1731916459522 (+408 ms)Flushing deeef93d343bf8d154ce5359480346cc/B: appending metadata at 1731916459524 (+2 ms)Flushing deeef93d343bf8d154ce5359480346cc/B: closing flushed file at 1731916459524Flushing deeef93d343bf8d154ce5359480346cc/C: creating writer at 1731916459931 (+407 ms)Flushing deeef93d343bf8d154ce5359480346cc/C: appending metadata at 1731916459933 (+2 ms)Flushing deeef93d343bf8d154ce5359480346cc/C: closing flushed file at 1731916459933Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d4f961: reopening flushed file at 1731916460340 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62899303: reopening flushed file at 1731916460351 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dca0d38: reopening flushed file at 1731916460356 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for deeef93d343bf8d154ce5359480346cc in 1253ms, sequenceid=531, compaction requested=true at 1731916460363 (+7 ms)Writing region close event to WAL at 1731916460433 (+70 ms)Running coprocessor post-close hooks at 1731916460436 (+3 ms)Closed at 1731916460436 2024-11-18T07:54:20,438 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:20,439 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=deeef93d343bf8d154ce5359480346cc, regionState=CLOSED 2024-11-18T07:54:20,440 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:54:20,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-18T07:54:20,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure deeef93d343bf8d154ce5359480346cc, server=87fb3d252950,37987,1731916323128 in 2.4800 sec 2024-11-18T07:54:20,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-18T07:54:20,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=deeef93d343bf8d154ce5359480346cc, UNASSIGN in 2.4850 sec 2024-11-18T07:54:20,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-18T07:54:20,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.4870 sec 2024-11-18T07:54:20,446 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916460446"}]},"ts":"1731916460446"} 2024-11-18T07:54:20,447 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T07:54:20,447 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T07:54:20,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.4960 sec 2024-11-18T07:54:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T07:54:22,118 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:22,119 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T07:54:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:54:22,135 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:54:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-18T07:54:22,147 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:54:22,161 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:22,165 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/recovered.edits] 2024-11-18T07:54:22,176 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/2af91fd273c04077983e2a6f5ee75d94 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/2af91fd273c04077983e2a6f5ee75d94 2024-11-18T07:54:22,181 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/58045fb6994b4a7f903953415d7c9efa to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/58045fb6994b4a7f903953415d7c9efa 2024-11-18T07:54:22,186 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/f250916cb6914473a392bd7bb5545e1b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/f250916cb6914473a392bd7bb5545e1b 2024-11-18T07:54:22,188 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/fe2d9879eb5743a8b0e467f91d6b33d0 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/A/fe2d9879eb5743a8b0e467f91d6b33d0 2024-11-18T07:54:22,196 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/09e36ac4e4c44f29b70ee9c96fe425c8 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/09e36ac4e4c44f29b70ee9c96fe425c8 2024-11-18T07:54:22,200 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/82e02e942bab4b65b8ac7768bd9cfaaf to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/82e02e942bab4b65b8ac7768bd9cfaaf 2024-11-18T07:54:22,204 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/e8816c609b4d429a8769d7cf26342fcc to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/e8816c609b4d429a8769d7cf26342fcc 2024-11-18T07:54:22,217 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/f19893d7d5314b4983e8a4770cba6b67 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/B/f19893d7d5314b4983e8a4770cba6b67 2024-11-18T07:54:22,228 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/24ccdc63b6634aefb0daeffffcaba58e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/24ccdc63b6634aefb0daeffffcaba58e 2024-11-18T07:54:22,231 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/85c6890d1db042de8c9c3567040f7c9c to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/85c6890d1db042de8c9c3567040f7c9c 2024-11-18T07:54:22,233 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/9d084d5efc414135912a5e5b38951b72 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/9d084d5efc414135912a5e5b38951b72 2024-11-18T07:54:22,235 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/f726e9a5505f4b4ea28a932c5e4b9967 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/C/f726e9a5505f4b4ea28a932c5e4b9967 2024-11-18T07:54:22,239 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/recovered.edits/534.seqid to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc/recovered.edits/534.seqid 2024-11-18T07:54:22,241 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/deeef93d343bf8d154ce5359480346cc 2024-11-18T07:54:22,242 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T07:54:22,247 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:54:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-18T07:54:22,249 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T07:54:22,252 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T07:54:22,253 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:54:22,253 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T07:54:22,254 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731916462253"}]},"ts":"9223372036854775807"} 2024-11-18T07:54:22,256 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T07:54:22,256 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => deeef93d343bf8d154ce5359480346cc, NAME => 'TestAcidGuarantees,,1731916436566.deeef93d343bf8d154ce5359480346cc.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T07:54:22,256 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T07:54:22,256 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731916462256"}]},"ts":"9223372036854775807"} 2024-11-18T07:54:22,258 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T07:54:22,262 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:54:22,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 143 msec 2024-11-18T07:54:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-18T07:54:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T07:54:22,458 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:22,470 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 252), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1010 (was 1105), ProcessCount=11 (was 11), AvailableMemoryMB=1794 (was 1846) 2024-11-18T07:54:22,484 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=1010, ProcessCount=11, AvailableMemoryMB=1794 2024-11-18T07:54:22,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T07:54:22,485 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T07:54:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T07:54:22,488 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T07:54:22,488 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:22,488 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-18T07:54:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T07:54:22,490 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T07:54:22,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742356_1532 (size=1061) 2024-11-18T07:54:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T07:54:22,725 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T07:54:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T07:54:22,926 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3 2024-11-18T07:54:22,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742357_1533 (size=53) 2024-11-18T07:54:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T07:54:23,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:54:23,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing aea54bdc27847a2fa7b24fdf7cbb20d6, disabling compactions & flushes 2024-11-18T07:54:23,332 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:23,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:23,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. after waiting 0 ms 2024-11-18T07:54:23,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:23,332 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:23,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for aea54bdc27847a2fa7b24fdf7cbb20d6: Waiting for close lock at 1731916463332Disabling compacts and flushes for region at 1731916463332Disabling writes for close at 1731916463332Writing region close event to WAL at 1731916463332Closed at 1731916463332 2024-11-18T07:54:23,334 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T07:54:23,334 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731916463334"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731916463334"}]},"ts":"1731916463334"} 2024-11-18T07:54:23,336 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T07:54:23,337 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T07:54:23,337 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916463337"}]},"ts":"1731916463337"} 2024-11-18T07:54:23,338 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T07:54:23,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea54bdc27847a2fa7b24fdf7cbb20d6, ASSIGN}] 2024-11-18T07:54:23,340 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea54bdc27847a2fa7b24fdf7cbb20d6, ASSIGN 2024-11-18T07:54:23,340 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea54bdc27847a2fa7b24fdf7cbb20d6, ASSIGN; state=OFFLINE, location=87fb3d252950,37987,1731916323128; forceNewPlan=false, retain=false 2024-11-18T07:54:23,492 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=aea54bdc27847a2fa7b24fdf7cbb20d6, regionState=OPENING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:54:23,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea54bdc27847a2fa7b24fdf7cbb20d6, ASSIGN because future has completed 2024-11-18T07:54:23,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:54:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T07:54:23,654 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:23,654 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} 2024-11-18T07:54:23,654 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:23,654 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:54:23,654 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:23,654 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:23,655 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:23,657 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:54:23,657 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aea54bdc27847a2fa7b24fdf7cbb20d6 columnFamilyName A 2024-11-18T07:54:23,657 DEBUG [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:23,658 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.HStore(327): Store=aea54bdc27847a2fa7b24fdf7cbb20d6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:54:23,658 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:23,659 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:54:23,659 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aea54bdc27847a2fa7b24fdf7cbb20d6 columnFamilyName B 2024-11-18T07:54:23,659 DEBUG [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:23,660 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.HStore(327): Store=aea54bdc27847a2fa7b24fdf7cbb20d6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:54:23,660 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:23,661 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:54:23,661 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aea54bdc27847a2fa7b24fdf7cbb20d6 columnFamilyName C 2024-11-18T07:54:23,661 DEBUG [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:23,662 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.HStore(327): Store=aea54bdc27847a2fa7b24fdf7cbb20d6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:54:23,662 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:23,662 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:23,663 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:23,663 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:23,664 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:23,664 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:23,665 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T07:54:23,666 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:23,668 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T07:54:23,668 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened aea54bdc27847a2fa7b24fdf7cbb20d6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67012915, jitterRate=-0.0014297515153884888}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T07:54:23,668 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:23,669 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for aea54bdc27847a2fa7b24fdf7cbb20d6: Running coprocessor pre-open hook at 1731916463654Writing region info on filesystem at 1731916463654Initializing all the Stores at 1731916463655 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916463655Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916463655Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916463655Cleaning up temporary data from old regions at 1731916463664 (+9 ms)Running coprocessor post-open hooks at 1731916463669 (+5 ms)Region opened successfully at 1731916463669 2024-11-18T07:54:23,670 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., pid=142, masterSystemTime=1731916463650 2024-11-18T07:54:23,672 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:23,672 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:23,672 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=aea54bdc27847a2fa7b24fdf7cbb20d6, regionState=OPEN, openSeqNum=2, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:54:23,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:54:23,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-18T07:54:23,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 in 177 msec 2024-11-18T07:54:23,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-18T07:54:23,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea54bdc27847a2fa7b24fdf7cbb20d6, ASSIGN in 338 msec 2024-11-18T07:54:23,682 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T07:54:23,683 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916463682"}]},"ts":"1731916463682"} 2024-11-18T07:54:23,684 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T07:54:23,685 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T07:54:23,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2000 sec 2024-11-18T07:54:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T07:54:24,638 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:24,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3071d898, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:24,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:54:24,640 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:54:24,641 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:54:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:54:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:54:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a80b38a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:54:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:54:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:24,643 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:54:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d3e7d79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:24,644 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:54:24,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:54:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:54:24,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:54:24,646 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T07:54:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 87fb3d252950,42633,1731916322278 2024-11-18T07:54:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5050f163 2024-11-18T07:54:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T07:54:24,648 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T07:54:24,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T07:54:24,649 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T07:54:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T07:54:24,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742358_1534 (size=1097) 2024-11-18T07:54:25,059 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-18T07:54:25,060 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-18T07:54:25,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T07:54:25,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea54bdc27847a2fa7b24fdf7cbb20d6, REOPEN/MOVE}] 2024-11-18T07:54:25,064 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea54bdc27847a2fa7b24fdf7cbb20d6, REOPEN/MOVE 2024-11-18T07:54:25,065 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=aea54bdc27847a2fa7b24fdf7cbb20d6, regionState=CLOSING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:54:25,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea54bdc27847a2fa7b24fdf7cbb20d6, REOPEN/MOVE because future has completed 2024-11-18T07:54:25,066 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T07:54:25,066 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:54:25,219 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:25,219 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T07:54:25,219 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing aea54bdc27847a2fa7b24fdf7cbb20d6, disabling compactions & flushes 2024-11-18T07:54:25,219 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:25,219 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:25,220 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. after waiting 0 ms 2024-11-18T07:54:25,220 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:25,223 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T07:54:25,224 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:25,224 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for aea54bdc27847a2fa7b24fdf7cbb20d6: Waiting for close lock at 1731916465219Running coprocessor pre-close hooks at 1731916465219Disabling compacts and flushes for region at 1731916465219Disabling writes for close at 1731916465220 (+1 ms)Writing region close event to WAL at 1731916465220Running coprocessor post-close hooks at 1731916465223 (+3 ms)Closed at 1731916465224 (+1 ms) 2024-11-18T07:54:25,224 WARN [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: aea54bdc27847a2fa7b24fdf7cbb20d6 to self. 2024-11-18T07:54:25,226 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:25,226 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=aea54bdc27847a2fa7b24fdf7cbb20d6, regionState=CLOSED 2024-11-18T07:54:25,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:54:25,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-18T07:54:25,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 in 163 msec 2024-11-18T07:54:25,231 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea54bdc27847a2fa7b24fdf7cbb20d6, REOPEN/MOVE; state=CLOSED, location=87fb3d252950,37987,1731916323128; forceNewPlan=false, retain=true 2024-11-18T07:54:25,382 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=aea54bdc27847a2fa7b24fdf7cbb20d6, regionState=OPENING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:54:25,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea54bdc27847a2fa7b24fdf7cbb20d6, REOPEN/MOVE because future has completed 2024-11-18T07:54:25,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:54:25,539 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:25,539 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} 2024-11-18T07:54:25,539 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:25,539 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T07:54:25,539 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:25,539 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:25,545 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:25,546 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:54:25,546 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aea54bdc27847a2fa7b24fdf7cbb20d6 columnFamilyName A 2024-11-18T07:54:25,547 DEBUG [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:25,548 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.HStore(327): Store=aea54bdc27847a2fa7b24fdf7cbb20d6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:54:25,548 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:25,549 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:54:25,549 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aea54bdc27847a2fa7b24fdf7cbb20d6 columnFamilyName B 2024-11-18T07:54:25,549 DEBUG [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:25,550 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.HStore(327): Store=aea54bdc27847a2fa7b24fdf7cbb20d6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:54:25,550 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:25,551 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T07:54:25,551 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aea54bdc27847a2fa7b24fdf7cbb20d6 columnFamilyName C 2024-11-18T07:54:25,551 DEBUG [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:25,551 INFO [StoreOpener-aea54bdc27847a2fa7b24fdf7cbb20d6-1 {}] regionserver.HStore(327): Store=aea54bdc27847a2fa7b24fdf7cbb20d6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T07:54:25,551 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:25,552 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:25,552 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:25,553 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:25,554 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:25,554 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:25,555 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T07:54:25,556 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:25,558 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened aea54bdc27847a2fa7b24fdf7cbb20d6; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68682465, jitterRate=0.023448482155799866}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T07:54:25,558 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:25,559 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for aea54bdc27847a2fa7b24fdf7cbb20d6: Running coprocessor pre-open hook at 1731916465540Writing region info on filesystem at 1731916465540Initializing all the Stores at 1731916465541 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916465541Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916465545 (+4 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731916465545Cleaning up temporary data from old regions at 1731916465554 (+9 ms)Running coprocessor post-open hooks at 1731916465558 (+4 ms)Region opened successfully at 1731916465559 (+1 ms) 2024-11-18T07:54:25,560 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., pid=147, masterSystemTime=1731916465536 2024-11-18T07:54:25,561 DEBUG [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:25,561 INFO [RS_OPEN_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:25,562 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=aea54bdc27847a2fa7b24fdf7cbb20d6, regionState=OPEN, openSeqNum=5, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:54:25,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:54:25,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-18T07:54:25,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 in 180 msec 2024-11-18T07:54:25,568 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-18T07:54:25,568 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea54bdc27847a2fa7b24fdf7cbb20d6, REOPEN/MOVE in 502 msec 2024-11-18T07:54:25,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-18T07:54:25,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 506 msec 2024-11-18T07:54:25,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 922 msec 2024-11-18T07:54:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-18T07:54:25,573 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:25,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a34ec22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:54:25,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:54:25,574 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:54:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:54:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:54:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e24ad5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:54:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:54:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:25,577 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:54:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b73b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@380ce272, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:54:25,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:54:25,580 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:54:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:54:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:54:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@120fe370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:54:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:54:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:25,582 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:54:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e240363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14c11d8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,583 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:54:25,583 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:54:25,584 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:54:25,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:54:25,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:54:25,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e1c148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:54:25,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:54:25,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:25,586 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:54:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7abb0159, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d301717, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:54:25,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:54:25,589 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:54:25,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:54:25,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:54:25,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19c15524, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:54:25,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:54:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:25,590 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:54:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d0bd84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,595 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36b0a686, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,595 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:54:25,595 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:54:25,596 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:54:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:54:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:54:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29867e4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:54:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:54:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:25,598 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:54:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46877111, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2966c360, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,603 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:54:25,603 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:54:25,604 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:54:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:54:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:54:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b0f2a65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:54:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:54:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:25,606 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:54:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@239f02b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6379f6de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,607 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:54:25,607 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:54:25,608 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:54:25,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:54:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:54:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23a58109, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:54:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:54:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:25,610 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:54:25,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6668303c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337ab365, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,623 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:54:25,623 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:54:25,624 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:54:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:54:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:54:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15865898, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:54:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:54:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:25,626 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:54:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20a79349, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e1be13a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,628 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:54:25,628 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:54:25,629 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:54:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:54:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:54:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29704a4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:54:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:54:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:25,632 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:54:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bd39d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fbae74f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb3d252950,42633,-1 for getting cluster id 2024-11-18T07:54:25,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T07:54:25,634 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46703b93-0193-4c59-a818-58ceb8c8d03b' 2024-11-18T07:54:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T07:54:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46703b93-0193-4c59-a818-58ceb8c8d03b" 2024-11-18T07:54:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5063eb05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb3d252950,42633,-1] 2024-11-18T07:54:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T07:54:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:25,636 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T07:54:25,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4a6589, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T07:54:25,638 DEBUG [Thread-2391 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:54:25,638 DEBUG [Thread-2382 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:54:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:54:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:54:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:54:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:54:25,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:54:25,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:54:25,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:54:25,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:54:25,651 DEBUG [Thread-2395 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:54:25,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:54:25,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:54:25,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:54:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:25,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:54:25,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:25,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:25,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:25,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:25,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:25,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:54:25,663 DEBUG [Thread-2397 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:54:25,666 DEBUG [Thread-2397 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:54:25,667 DEBUG [Thread-2397 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:54:25,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:54:25,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:54:25,675 DEBUG [Thread-2388 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:54:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:54:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:54:25,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:54:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:54:25,686 DEBUG [Thread-2399 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:54:25,686 DEBUG [Thread-2380 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:54:25,687 DEBUG [Thread-2393 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:54:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:54:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:54:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:54:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:54:25,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:54:25,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:54:25,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:54:25,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118597e0457f88e4bc3a1dfc919f00247b1_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916465650/Put/seqid=0 2024-11-18T07:54:25,689 DEBUG [Thread-2386 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:54:25,689 DEBUG [Thread-2384 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T07:54:25,690 DEBUG [Thread-2386 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:54:25,690 DEBUG [Thread-2386 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:54:25,690 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:54:25,691 DEBUG [Thread-2384 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb3d252950,37987,1731916323128, seqNum=-1] 2024-11-18T07:54:25,691 DEBUG [Thread-2384 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T07:54:25,691 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:54:25,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:54:25,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:54:25,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:54:25,693 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:54:25,694 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:54:25,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T07:54:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:54:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-18T07:54:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T07:54:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5] 2024-11-18T07:54:25,697 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:54:25,699 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:54:25,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:25,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742359_1535 (size=12154) 2024-11-18T07:54:25,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916465700, max timestamp=1731916465720Num uniques 859; 2024-11-18T07:54:25,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916465700, max timestamp=1731916465720Num uniques 859; 2024-11-18T07:54:25,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:25,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:25,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916465700, max timestamp=1731916465720Num uniques 859; 2024-11-18T07:54:25,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:25,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916525759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:25,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:25,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916525760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:25,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916525761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:25,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916525763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:25,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916525764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T07:54:25,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T07:54:25,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:25,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:25,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:25,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:25,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:25,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916525879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:25,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916525879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916525879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:25,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916525879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:25,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916525879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:25,955 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T07:54:26,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T07:54:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T07:54:26,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:26,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:26,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:26,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916526098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:26,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:26,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916526098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:26,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916526098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:26,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916526098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916526098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:26,115 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:26,145 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118597e0457f88e4bc3a1dfc919f00247b1_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118597e0457f88e4bc3a1dfc919f00247b1_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:26,152 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/2e2cd06a91204d6d9b32bec8700f95ca, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:26,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/2e2cd06a91204d6d9b32bec8700f95ca is 175, key is test_row_0/A:col10/1731916465650/Put/seqid=0 2024-11-18T07:54:26,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T07:54:26,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:26,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:26,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:26,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742360_1536 (size=30955) 2024-11-18T07:54:26,185 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/2e2cd06a91204d6d9b32bec8700f95ca 2024-11-18T07:54:26,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/6ae2c6b8f79e413bbe7c4313b1765bd4 is 50, key is test_row_0/B:col10/1731916465650/Put/seqid=0 2024-11-18T07:54:26,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742361_1537 (size=12001) 2024-11-18T07:54:26,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T07:54:26,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:26,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:26,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T07:54:26,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916526409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:26,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916526409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916526413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:26,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916526413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:26,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916526413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:26,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T07:54:26,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:26,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:26,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:26,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T07:54:26,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:26,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:26,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:26,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/6ae2c6b8f79e413bbe7c4313b1765bd4 2024-11-18T07:54:26,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/d892a5dd86e2400694613168fe11a6fc is 50, key is test_row_0/C:col10/1731916465650/Put/seqid=0 2024-11-18T07:54:26,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T07:54:26,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:26,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:26,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:26,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742362_1538 (size=12001) 2024-11-18T07:54:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T07:54:26,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:26,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916526928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916526928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:26,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:26,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916526928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916526928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916526929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:26,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T07:54:26,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:26,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:26,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:26,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:27,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T07:54:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:27,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:27,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/d892a5dd86e2400694613168fe11a6fc 2024-11-18T07:54:27,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/2e2cd06a91204d6d9b32bec8700f95ca as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2e2cd06a91204d6d9b32bec8700f95ca 2024-11-18T07:54:27,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2e2cd06a91204d6d9b32bec8700f95ca, entries=150, sequenceid=14, filesize=30.2 K 2024-11-18T07:54:27,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/6ae2c6b8f79e413bbe7c4313b1765bd4 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/6ae2c6b8f79e413bbe7c4313b1765bd4 2024-11-18T07:54:27,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/6ae2c6b8f79e413bbe7c4313b1765bd4, entries=150, sequenceid=14, filesize=11.7 K 2024-11-18T07:54:27,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/d892a5dd86e2400694613168fe11a6fc as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/d892a5dd86e2400694613168fe11a6fc 2024-11-18T07:54:27,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/d892a5dd86e2400694613168fe11a6fc, entries=150, sequenceid=14, filesize=11.7 K 2024-11-18T07:54:27,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 1561ms, sequenceid=14, compaction requested=false 2024-11-18T07:54:27,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:27,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T07:54:27,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:27,260 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-18T07:54:27,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:27,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182eeedb401c504394bdc5e0869ee7a875_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916465758/Put/seqid=0 2024-11-18T07:54:27,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742363_1539 (size=12154) 2024-11-18T07:54:27,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,343 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182eeedb401c504394bdc5e0869ee7a875_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182eeedb401c504394bdc5e0869ee7a875_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:27,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/8700a36212764fd5ab7cf167c5a968c4, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:27,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/8700a36212764fd5ab7cf167c5a968c4 is 175, key is test_row_0/A:col10/1731916465758/Put/seqid=0 2024-11-18T07:54:27,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742364_1540 (size=30955) 2024-11-18T07:54:27,376 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/8700a36212764fd5ab7cf167c5a968c4 2024-11-18T07:54:27,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/8e984a70571d4c9ea453867bbf4f6e3f is 50, key is test_row_0/B:col10/1731916465758/Put/seqid=0 2024-11-18T07:54:27,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742365_1541 (size=12001) 2024-11-18T07:54:27,449 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/8e984a70571d4c9ea453867bbf4f6e3f 2024-11-18T07:54:27,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/05d876d84187410fa0d4288ccc7163fe is 50, key is test_row_0/C:col10/1731916465758/Put/seqid=0 2024-11-18T07:54:27,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742366_1542 (size=12001) 2024-11-18T07:54:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T07:54:27,939 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/05d876d84187410fa0d4288ccc7163fe 2024-11-18T07:54:27,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/8700a36212764fd5ab7cf167c5a968c4 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/8700a36212764fd5ab7cf167c5a968c4 2024-11-18T07:54:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:27,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:27,948 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/8700a36212764fd5ab7cf167c5a968c4, entries=150, sequenceid=51, filesize=30.2 K 2024-11-18T07:54:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/8e984a70571d4c9ea453867bbf4f6e3f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8e984a70571d4c9ea453867bbf4f6e3f 2024-11-18T07:54:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,956 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8e984a70571d4c9ea453867bbf4f6e3f, entries=150, sequenceid=51, filesize=11.7 K 2024-11-18T07:54:27,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/05d876d84187410fa0d4288ccc7163fe as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/05d876d84187410fa0d4288ccc7163fe 2024-11-18T07:54:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,962 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/05d876d84187410fa0d4288ccc7163fe, entries=150, sequenceid=51, filesize=11.7 K 2024-11-18T07:54:27,963 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=26.84 KB/27480 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 703ms, sequenceid=51, compaction requested=false 2024-11-18T07:54:27,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:27,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:27,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-18T07:54:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-18T07:54:27,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-18T07:54:27,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2650 sec 2024-11-18T07:54:27,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.2730 sec 2024-11-18T07:54:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:28,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:54:28,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:28,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:28,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:28,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:28,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118024f953178ac4aae9f42db596d40f55c_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916468000/Put/seqid=0 2024-11-18T07:54:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916468064, max timestamp=1731916468079Num uniques 859; 2024-11-18T07:54:28,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:28,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916468064, max timestamp=1731916468079Num uniques 859; 2024-11-18T07:54:28,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:28,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916468064, max timestamp=1731916468079Num uniques 859; 2024-11-18T07:54:28,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:28,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742367_1543 (size=17034) 2024-11-18T07:54:28,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916528163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916528165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731916468161, max timestamp=1731916468183Num uniques 1068; 2024-11-18T07:54:28,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731916468161, max timestamp=1731916468183Num uniques 1068; 2024-11-18T07:54:28,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-18T07:54:28,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-18T07:54:28,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731916468161, max timestamp=1731916468183Num uniques 1018; 2024-11-18T07:54:28,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T07:54:28,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916528283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731916468260, max timestamp=1731916468297Num uniques 1027; 2024-11-18T07:54:28,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-18T07:54:28,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731916468260, max timestamp=1731916468297Num uniques 1027; 2024-11-18T07:54:28,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731916468260, max timestamp=1731916468297Num uniques 1027; 2024-11-18T07:54:28,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-18T07:54:28,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-18T07:54:28,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916528354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916528359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916528360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916528362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916528408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916528468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916528468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916528479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916528479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,496 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:28,498 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118024f953178ac4aae9f42db596d40f55c_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118024f953178ac4aae9f42db596d40f55c_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:28,499 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/c508216608fd4e29953372b261387b7a, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:28,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/c508216608fd4e29953372b261387b7a is 175, key is test_row_0/A:col10/1731916468000/Put/seqid=0 2024-11-18T07:54:28,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742368_1544 (size=48139) 2024-11-18T07:54:28,552 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/c508216608fd4e29953372b261387b7a 2024-11-18T07:54:28,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/0594a3b6c76f4ce1855aba42e64c1076 is 50, key is test_row_0/B:col10/1731916468000/Put/seqid=0 2024-11-18T07:54:28,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742369_1545 (size=12001) 2024-11-18T07:54:28,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916528618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916528678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916528678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916528688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916528688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916528930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/0594a3b6c76f4ce1855aba42e64c1076 2024-11-18T07:54:28,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/dd4081404b54431fb8196547920df492 is 50, key is test_row_0/C:col10/1731916468000/Put/seqid=0 2024-11-18T07:54:28,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742370_1546 (size=12001) 2024-11-18T07:54:28,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916528988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916528989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916528998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916529008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/dd4081404b54431fb8196547920df492 2024-11-18T07:54:29,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/c508216608fd4e29953372b261387b7a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/c508216608fd4e29953372b261387b7a 2024-11-18T07:54:29,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/c508216608fd4e29953372b261387b7a, entries=250, sequenceid=61, filesize=47.0 K 2024-11-18T07:54:29,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/0594a3b6c76f4ce1855aba42e64c1076 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/0594a3b6c76f4ce1855aba42e64c1076 2024-11-18T07:54:29,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/0594a3b6c76f4ce1855aba42e64c1076, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T07:54:29,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/dd4081404b54431fb8196547920df492 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/dd4081404b54431fb8196547920df492 2024-11-18T07:54:29,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/dd4081404b54431fb8196547920df492, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T07:54:29,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 1409ms, sequenceid=61, compaction requested=true 2024-11-18T07:54:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:29,411 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:29,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:29,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:29,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:29,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:29,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:29,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:54:29,411 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:29,413 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:29,413 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:29,413 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/B is initiating minor compaction (all files) 2024-11-18T07:54:29,413 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/A is initiating minor compaction (all files) 2024-11-18T07:54:29,413 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/B in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:29,413 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/A in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:29,413 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2e2cd06a91204d6d9b32bec8700f95ca, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/8700a36212764fd5ab7cf167c5a968c4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/c508216608fd4e29953372b261387b7a] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=107.5 K 2024-11-18T07:54:29,413 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/6ae2c6b8f79e413bbe7c4313b1765bd4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8e984a70571d4c9ea453867bbf4f6e3f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/0594a3b6c76f4ce1855aba42e64c1076] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=35.2 K 2024-11-18T07:54:29,413 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:29,413 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2e2cd06a91204d6d9b32bec8700f95ca, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/8700a36212764fd5ab7cf167c5a968c4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/c508216608fd4e29953372b261387b7a] 2024-11-18T07:54:29,416 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e2cd06a91204d6d9b32bec8700f95ca, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731916465645 2024-11-18T07:54:29,416 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ae2c6b8f79e413bbe7c4313b1765bd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731916465645 2024-11-18T07:54:29,417 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8700a36212764fd5ab7cf167c5a968c4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731916465758 2024-11-18T07:54:29,418 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e984a70571d4c9ea453867bbf4f6e3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731916465758 2024-11-18T07:54:29,418 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c508216608fd4e29953372b261387b7a, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731916467945 2024-11-18T07:54:29,418 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0594a3b6c76f4ce1855aba42e64c1076, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731916467945 2024-11-18T07:54:29,429 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:29,431 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#B#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:29,432 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/8490c4a4966b443586c0c7217eecc246 is 50, key is test_row_0/B:col10/1731916468000/Put/seqid=0 2024-11-18T07:54:29,433 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111821d0b2be5d104a52a89487c7506fb409_aea54bdc27847a2fa7b24fdf7cbb20d6 store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:29,435 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111821d0b2be5d104a52a89487c7506fb409_aea54bdc27847a2fa7b24fdf7cbb20d6, store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:29,435 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111821d0b2be5d104a52a89487c7506fb409_aea54bdc27847a2fa7b24fdf7cbb20d6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:29,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731916468354, max timestamp=1731916469457Num uniques 1036; 2024-11-18T07:54:29,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731916468354, max timestamp=1731916469457Num uniques 1036; 2024-11-18T07:54:29,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-18T07:54:29,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-18T07:54:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:29,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=64.85 KB heapSize=177.11 KB 2024-11-18T07:54:29,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:29,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:29,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:29,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:29,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:29,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T07:54:29,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742371_1547 (size=12104) 2024-11-18T07:54:29,490 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/8490c4a4966b443586c0c7217eecc246 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8490c4a4966b443586c0c7217eecc246 2024-11-18T07:54:29,495 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/B of aea54bdc27847a2fa7b24fdf7cbb20d6 into 8490c4a4966b443586c0c7217eecc246(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:29,495 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:29,495 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/B, priority=13, startTime=1731916469411; duration=0sec 2024-11-18T07:54:29,495 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:29,495 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:B 2024-11-18T07:54:29,495 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:29,496 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:29,496 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/C is initiating minor compaction (all files) 2024-11-18T07:54:29,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742372_1548 (size=4469) 2024-11-18T07:54:29,499 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/C in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:29,499 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/d892a5dd86e2400694613168fe11a6fc, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/05d876d84187410fa0d4288ccc7163fe, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/dd4081404b54431fb8196547920df492] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=35.2 K 2024-11-18T07:54:29,500 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting d892a5dd86e2400694613168fe11a6fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731916465645 2024-11-18T07:54:29,500 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#A#compaction#467 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:29,500 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/44705e3f7977439f91449ad8cc66e96b is 175, key is test_row_0/A:col10/1731916468000/Put/seqid=0 2024-11-18T07:54:29,501 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 05d876d84187410fa0d4288ccc7163fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731916465758 2024-11-18T07:54:29,501 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting dd4081404b54431fb8196547920df492, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731916467945 2024-11-18T07:54:29,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180ff1ed5e928d4b0d8d1ccffec7a58165_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916468355/Put/seqid=0 2024-11-18T07:54:29,537 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#C#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:29,537 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/c878340e11414e0b8a54c94204bedc95 is 50, key is test_row_0/C:col10/1731916468000/Put/seqid=0 2024-11-18T07:54:29,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916529545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916529547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916529545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916529550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916529563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742373_1549 (size=31058) 2024-11-18T07:54:29,584 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/44705e3f7977439f91449ad8cc66e96b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/44705e3f7977439f91449ad8cc66e96b 2024-11-18T07:54:29,590 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/A of aea54bdc27847a2fa7b24fdf7cbb20d6 into 44705e3f7977439f91449ad8cc66e96b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:29,590 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:29,590 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/A, priority=13, startTime=1731916469410; duration=0sec 2024-11-18T07:54:29,590 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:29,590 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:A 2024-11-18T07:54:29,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742374_1550 (size=14794) 2024-11-18T07:54:29,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742375_1551 (size=12104) 2024-11-18T07:54:29,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916529668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916529668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916529668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916529678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916529679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T07:54:29,860 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:29,861 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:54:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-18T07:54:29,862 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:54:29,863 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:54:29,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:29,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T07:54:29,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916529888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916529889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916529889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916529889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916529888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T07:54:30,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T07:54:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:30,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,023 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:30,029 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180ff1ed5e928d4b0d8d1ccffec7a58165_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180ff1ed5e928d4b0d8d1ccffec7a58165_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:30,030 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/b4793be7203b443ab7dc1f0eb27fb433, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:30,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/b4793be7203b443ab7dc1f0eb27fb433 is 175, key is test_row_0/A:col10/1731916468355/Put/seqid=0 2024-11-18T07:54:30,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742376_1552 (size=39749) 2024-11-18T07:54:30,055 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/c878340e11414e0b8a54c94204bedc95 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/c878340e11414e0b8a54c94204bedc95 2024-11-18T07:54:30,062 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/C of aea54bdc27847a2fa7b24fdf7cbb20d6 into c878340e11414e0b8a54c94204bedc95(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:30,062 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:30,062 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/C, priority=13, startTime=1731916469411; duration=0sec 2024-11-18T07:54:30,062 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:30,062 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:C 2024-11-18T07:54:30,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T07:54:30,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:30,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:30,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:30,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T07:54:30,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916530198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:30,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916530198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:30,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916530208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:30,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:30,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916530208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:30,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:30,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916530208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:30,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T07:54:30,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:30,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:30,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:30,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,451 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/b4793be7203b443ab7dc1f0eb27fb433 2024-11-18T07:54:30,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/55a134e775684fa9875d6d0f81c6167b is 50, key is test_row_0/B:col10/1731916468355/Put/seqid=0 2024-11-18T07:54:30,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T07:54:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:30,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T07:54:30,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742377_1553 (size=12151) 2024-11-18T07:54:30,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T07:54:30,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:30,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:30,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:30,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916530710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:30,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916530711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:30,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916530719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:30,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916530719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:30,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916530719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:30,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T07:54:30,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:30,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:30,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:30,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:30,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/55a134e775684fa9875d6d0f81c6167b 2024-11-18T07:54:30,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/3924ea7bffe740a8af25c5df5dc53e25 is 50, key is test_row_0/C:col10/1731916468355/Put/seqid=0 2024-11-18T07:54:30,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742378_1554 (size=12151) 2024-11-18T07:54:30,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/3924ea7bffe740a8af25c5df5dc53e25 2024-11-18T07:54:30,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/b4793be7203b443ab7dc1f0eb27fb433 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/b4793be7203b443ab7dc1f0eb27fb433 2024-11-18T07:54:30,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/b4793be7203b443ab7dc1f0eb27fb433, entries=200, sequenceid=133, filesize=38.8 K 2024-11-18T07:54:30,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/55a134e775684fa9875d6d0f81c6167b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/55a134e775684fa9875d6d0f81c6167b 2024-11-18T07:54:30,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/55a134e775684fa9875d6d0f81c6167b, entries=150, sequenceid=133, filesize=11.9 K 2024-11-18T07:54:30,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/3924ea7bffe740a8af25c5df5dc53e25 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/3924ea7bffe740a8af25c5df5dc53e25 2024-11-18T07:54:30,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/3924ea7bffe740a8af25c5df5dc53e25, entries=150, sequenceid=133, filesize=11.9 K 2024-11-18T07:54:30,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~177.06 KB/181312, currentSize=114.05 KB/116790 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 1481ms, sequenceid=133, compaction requested=false 2024-11-18T07:54:30,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:30,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T07:54:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:30,951 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T07:54:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:30,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:30,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:30,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:30,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118cff1e57fd0c640fd896b203b33fbbb9f_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916469548/Put/seqid=0 2024-11-18T07:54:30,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742379_1555 (size=12304) 2024-11-18T07:54:30,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:30,980 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118cff1e57fd0c640fd896b203b33fbbb9f_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118cff1e57fd0c640fd896b203b33fbbb9f_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:30,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/167a80525ab3465290df8a736a060440, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:30,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/167a80525ab3465290df8a736a060440 is 175, key is test_row_0/A:col10/1731916469548/Put/seqid=0 2024-11-18T07:54:30,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742380_1556 (size=31105) 2024-11-18T07:54:30,997 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/167a80525ab3465290df8a736a060440 2024-11-18T07:54:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T07:54:31,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/8649c88c851b4694b1371e13bd8115a7 is 50, key is test_row_0/B:col10/1731916469548/Put/seqid=0 2024-11-18T07:54:31,043 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T07:54:31,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742381_1557 (size=12151) 2024-11-18T07:54:31,236 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T07:54:31,483 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/8649c88c851b4694b1371e13bd8115a7 2024-11-18T07:54:31,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/e7bfbc09a2ab4a59a8172d887cc9b2d5 is 50, key is test_row_0/C:col10/1731916469548/Put/seqid=0 2024-11-18T07:54:31,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742382_1558 (size=12151) 2024-11-18T07:54:31,535 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/e7bfbc09a2ab4a59a8172d887cc9b2d5 2024-11-18T07:54:31,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/167a80525ab3465290df8a736a060440 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/167a80525ab3465290df8a736a060440 2024-11-18T07:54:31,588 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/167a80525ab3465290df8a736a060440, entries=150, sequenceid=156, filesize=30.4 K 2024-11-18T07:54:31,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/8649c88c851b4694b1371e13bd8115a7 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8649c88c851b4694b1371e13bd8115a7 2024-11-18T07:54:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,593 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8649c88c851b4694b1371e13bd8115a7, entries=150, sequenceid=156, filesize=11.9 K 2024-11-18T07:54:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/e7bfbc09a2ab4a59a8172d887cc9b2d5 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/e7bfbc09a2ab4a59a8172d887cc9b2d5 2024-11-18T07:54:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,598 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/e7bfbc09a2ab4a59a8172d887cc9b2d5, entries=150, sequenceid=156, filesize=11.9 K 2024-11-18T07:54:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,599 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 647ms, sequenceid=156, compaction requested=true 2024-11-18T07:54:31,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-18T07:54:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-18T07:54:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-18T07:54:31,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7370 sec 2024-11-18T07:54:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.7420 sec 2024-11-18T07:54:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:31,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:54:31,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:31,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:31,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:31,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180a6a79a3b43a40e08b7183ffe2ae87e3_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916471771/Put/seqid=0 2024-11-18T07:54:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731916471811, max timestamp=1731916471839Num uniques 859; 2024-11-18T07:54:31,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T07:54:31,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916471811, max timestamp=1731916471839Num uniques 859; 2024-11-18T07:54:31,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:31,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916471811, max timestamp=1731916471839Num uniques 859; 2024-11-18T07:54:31,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:31,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742383_1559 (size=19774) 2024-11-18T07:54:31,850 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:31,859 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180a6a79a3b43a40e08b7183ffe2ae87e3_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180a6a79a3b43a40e08b7183ffe2ae87e3_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:31,860 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/dfed1b4e06304ccfbee4518cc6e38017, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:31,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/dfed1b4e06304ccfbee4518cc6e38017 is 175, key is test_row_0/A:col10/1731916471771/Put/seqid=0 2024-11-18T07:54:31,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742384_1560 (size=57029) 2024-11-18T07:54:31,885 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/dfed1b4e06304ccfbee4518cc6e38017 2024-11-18T07:54:31,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/82cad5ed754448cdbfaed210c329f0fa is 50, key is test_row_0/B:col10/1731916471771/Put/seqid=0 2024-11-18T07:54:31,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:31,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916531902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:31,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742385_1561 (size=9757) 2024-11-18T07:54:31,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/82cad5ed754448cdbfaed210c329f0fa 2024-11-18T07:54:31,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:31,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916531920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:31,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:31,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:31,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:31,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:31,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/b294e05fd531467583cb338128eb1e2b is 50, key is test_row_0/C:col10/1731916471771/Put/seqid=0 2024-11-18T07:54:31,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916531928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:31,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:31,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:31,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:31,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916531929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:31,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:31,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916531929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:31,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:31,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:31,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:31,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742386_1562 (size=9757) 2024-11-18T07:54:32,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T07:54:32,019 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:32,019 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:54:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-18T07:54:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T07:54:32,021 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:54:32,022 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:54:32,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:32,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916532038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916532058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916532058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916532058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916532058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T07:54:32,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T07:54:32,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:32,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:32,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:32,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:32,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:32,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916532258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916532278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916532288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916532288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916532288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T07:54:32,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:32,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:32,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:32,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:32,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T07:54:32,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/b294e05fd531467583cb338128eb1e2b 2024-11-18T07:54:32,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/dfed1b4e06304ccfbee4518cc6e38017 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/dfed1b4e06304ccfbee4518cc6e38017 2024-11-18T07:54:32,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/dfed1b4e06304ccfbee4518cc6e38017, entries=300, sequenceid=166, filesize=55.7 K 2024-11-18T07:54:32,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/82cad5ed754448cdbfaed210c329f0fa as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/82cad5ed754448cdbfaed210c329f0fa 2024-11-18T07:54:32,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/82cad5ed754448cdbfaed210c329f0fa, entries=100, sequenceid=166, filesize=9.5 K 2024-11-18T07:54:32,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/b294e05fd531467583cb338128eb1e2b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b294e05fd531467583cb338128eb1e2b 2024-11-18T07:54:32,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b294e05fd531467583cb338128eb1e2b, entries=100, sequenceid=166, filesize=9.5 K 2024-11-18T07:54:32,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 621ms, sequenceid=166, compaction requested=true 2024-11-18T07:54:32,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:32,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:32,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:32,407 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:54:32,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:32,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:32,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:32,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T07:54:32,407 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:54:32,409 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158941 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:54:32,409 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/A is initiating minor compaction (all files) 2024-11-18T07:54:32,409 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/A in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:32,409 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/44705e3f7977439f91449ad8cc66e96b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/b4793be7203b443ab7dc1f0eb27fb433, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/167a80525ab3465290df8a736a060440, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/dfed1b4e06304ccfbee4518cc6e38017] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=155.2 K 2024-11-18T07:54:32,409 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:32,409 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/44705e3f7977439f91449ad8cc66e96b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/b4793be7203b443ab7dc1f0eb27fb433, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/167a80525ab3465290df8a736a060440, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/dfed1b4e06304ccfbee4518cc6e38017] 2024-11-18T07:54:32,409 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46163 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:54:32,409 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/B is initiating minor compaction (all files) 2024-11-18T07:54:32,409 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/B in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:32,409 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44705e3f7977439f91449ad8cc66e96b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731916467945 2024-11-18T07:54:32,409 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8490c4a4966b443586c0c7217eecc246, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/55a134e775684fa9875d6d0f81c6167b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8649c88c851b4694b1371e13bd8115a7, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/82cad5ed754448cdbfaed210c329f0fa] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=45.1 K 2024-11-18T07:54:32,410 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4793be7203b443ab7dc1f0eb27fb433, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731916468354 2024-11-18T07:54:32,410 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8490c4a4966b443586c0c7217eecc246, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731916467945 2024-11-18T07:54:32,410 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 167a80525ab3465290df8a736a060440, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731916469532 2024-11-18T07:54:32,410 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 55a134e775684fa9875d6d0f81c6167b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731916468355 2024-11-18T07:54:32,410 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfed1b4e06304ccfbee4518cc6e38017, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731916471758 2024-11-18T07:54:32,410 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8649c88c851b4694b1371e13bd8115a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731916469532 2024-11-18T07:54:32,410 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 82cad5ed754448cdbfaed210c329f0fa, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731916471771 2024-11-18T07:54:32,438 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#B#compaction#479 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:32,438 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/e261f20ea3644328a4c9c7118a4d32df is 50, key is test_row_0/B:col10/1731916471771/Put/seqid=0 2024-11-18T07:54:32,447 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:32,467 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118ddfde0c61411412490d69b178bb42f2f_aea54bdc27847a2fa7b24fdf7cbb20d6 store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:32,470 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118ddfde0c61411412490d69b178bb42f2f_aea54bdc27847a2fa7b24fdf7cbb20d6, store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:32,470 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ddfde0c61411412490d69b178bb42f2f_aea54bdc27847a2fa7b24fdf7cbb20d6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:32,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742387_1563 (size=12391) 2024-11-18T07:54:32,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T07:54:32,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:32,482 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-11-18T07:54:32,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:32,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:32,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:32,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:32,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:32,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:32,490 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/e261f20ea3644328a4c9c7118a4d32df as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/e261f20ea3644328a4c9c7118a4d32df 2024-11-18T07:54:32,497 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/B of aea54bdc27847a2fa7b24fdf7cbb20d6 into e261f20ea3644328a4c9c7118a4d32df(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:32,497 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:32,497 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/B, priority=12, startTime=1731916472407; duration=0sec 2024-11-18T07:54:32,497 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:32,498 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:B 2024-11-18T07:54:32,498 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:54:32,499 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46163 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:54:32,500 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/C is initiating minor compaction (all files) 2024-11-18T07:54:32,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742388_1564 (size=4469) 2024-11-18T07:54:32,500 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/C in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:32,500 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/c878340e11414e0b8a54c94204bedc95, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/3924ea7bffe740a8af25c5df5dc53e25, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/e7bfbc09a2ab4a59a8172d887cc9b2d5, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b294e05fd531467583cb338128eb1e2b] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=45.1 K 2024-11-18T07:54:32,500 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting c878340e11414e0b8a54c94204bedc95, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731916467945 2024-11-18T07:54:32,501 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3924ea7bffe740a8af25c5df5dc53e25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731916468355 2024-11-18T07:54:32,501 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting e7bfbc09a2ab4a59a8172d887cc9b2d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731916469532 2024-11-18T07:54:32,502 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting b294e05fd531467583cb338128eb1e2b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731916471771 2024-11-18T07:54:32,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f4628cc6241d47939cf98e8d37404705_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916471913/Put/seqid=0 2024-11-18T07:54:32,546 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#C#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:32,547 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/b579bb4fc91841a6b72a7dd71a0cf248 is 50, key is test_row_0/C:col10/1731916471771/Put/seqid=0 2024-11-18T07:54:32,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742389_1565 (size=12304) 2024-11-18T07:54:32,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:32,585 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f4628cc6241d47939cf98e8d37404705_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f4628cc6241d47939cf98e8d37404705_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:32,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/622ce6960eaf4afa9334c46982e7b331, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:32,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/622ce6960eaf4afa9334c46982e7b331 is 175, key is test_row_0/A:col10/1731916471913/Put/seqid=0 2024-11-18T07:54:32,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:32,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742390_1566 (size=12391) 2024-11-18T07:54:32,606 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/b579bb4fc91841a6b72a7dd71a0cf248 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b579bb4fc91841a6b72a7dd71a0cf248 2024-11-18T07:54:32,617 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/C of aea54bdc27847a2fa7b24fdf7cbb20d6 into b579bb4fc91841a6b72a7dd71a0cf248(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:32,617 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:32,617 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/C, priority=12, startTime=1731916472407; duration=0sec 2024-11-18T07:54:32,617 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:32,617 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:C 2024-11-18T07:54:32,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916532607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916532608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916532613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916532618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916532618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742391_1567 (size=31105) 2024-11-18T07:54:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T07:54:32,725 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T07:54:32,725 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-18T07:54:32,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916532728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916532729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916532729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916532738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916532738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,902 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#A#compaction#480 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:32,903 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/6e522757facf443e855c8b8e607cd402 is 175, key is test_row_0/A:col10/1731916471771/Put/seqid=0 2024-11-18T07:54:32,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742392_1568 (size=31452) 2024-11-18T07:54:32,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916532938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916532938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916532938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916532949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916532951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:32,961 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/6e522757facf443e855c8b8e607cd402 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/6e522757facf443e855c8b8e607cd402 2024-11-18T07:54:32,971 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/A of aea54bdc27847a2fa7b24fdf7cbb20d6 into 6e522757facf443e855c8b8e607cd402(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:32,971 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:32,971 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/A, priority=12, startTime=1731916472407; duration=0sec 2024-11-18T07:54:32,971 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:32,971 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:A 2024-11-18T07:54:33,033 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/622ce6960eaf4afa9334c46982e7b331 2024-11-18T07:54:33,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/b28109b7d9e04aca82b953f85c9fffac is 50, key is test_row_0/B:col10/1731916471913/Put/seqid=0 2024-11-18T07:54:33,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742393_1569 (size=12151) 2024-11-18T07:54:33,123 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/b28109b7d9e04aca82b953f85c9fffac 2024-11-18T07:54:33,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/0a506225b9264e30b20813371ec91d4e is 50, key is test_row_0/C:col10/1731916471913/Put/seqid=0 2024-11-18T07:54:33,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742394_1570 (size=12151) 2024-11-18T07:54:33,144 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/0a506225b9264e30b20813371ec91d4e 2024-11-18T07:54:33,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/622ce6960eaf4afa9334c46982e7b331 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/622ce6960eaf4afa9334c46982e7b331 2024-11-18T07:54:33,150 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/622ce6960eaf4afa9334c46982e7b331, entries=150, sequenceid=202, filesize=30.4 K 2024-11-18T07:54:33,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/b28109b7d9e04aca82b953f85c9fffac as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b28109b7d9e04aca82b953f85c9fffac 2024-11-18T07:54:33,155 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b28109b7d9e04aca82b953f85c9fffac, entries=150, sequenceid=202, filesize=11.9 K 2024-11-18T07:54:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T07:54:33,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/0a506225b9264e30b20813371ec91d4e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/0a506225b9264e30b20813371ec91d4e 2024-11-18T07:54:33,167 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/0a506225b9264e30b20813371ec91d4e, entries=150, sequenceid=202, filesize=11.9 K 2024-11-18T07:54:33,168 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=46.96 KB/48090 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 686ms, sequenceid=202, compaction requested=false 2024-11-18T07:54:33,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:33,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:33,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-18T07:54:33,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-18T07:54:33,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-18T07:54:33,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1480 sec 2024-11-18T07:54:33,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.1600 sec 2024-11-18T07:54:33,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:33,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T07:54:33,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:33,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:33,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:33,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:33,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:33,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:33,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:33,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916533295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:33,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:33,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916533299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:33,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:33,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916533299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:33,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916533302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:33,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:33,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916533302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:33,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111866ef979fe78243d7af2263135e86de30_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916472615/Put/seqid=0 2024-11-18T07:54:33,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742395_1571 (size=12304) 2024-11-18T07:54:33,342 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:33,345 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111866ef979fe78243d7af2263135e86de30_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111866ef979fe78243d7af2263135e86de30_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:33,349 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/2c875fe1192b4b75932fa35a50a8f411, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:33,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/2c875fe1192b4b75932fa35a50a8f411 is 175, key is test_row_0/A:col10/1731916472615/Put/seqid=0 2024-11-18T07:54:33,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742396_1572 (size=31105) 2024-11-18T07:54:33,393 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/2c875fe1192b4b75932fa35a50a8f411 2024-11-18T07:54:33,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/a1b9e8d0873c40d983a3a4b9a53f3fa4 is 50, key is test_row_0/B:col10/1731916472615/Put/seqid=0 2024-11-18T07:54:33,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:33,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916533419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916533421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:33,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916533421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:33,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916533421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:33,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:33,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916533422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:33,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742397_1573 (size=12151) 2024-11-18T07:54:33,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/a1b9e8d0873c40d983a3a4b9a53f3fa4 2024-11-18T07:54:33,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/ff8d30b8ea684a699bf2e61303b7ddba is 50, key is test_row_0/C:col10/1731916472615/Put/seqid=0 2024-11-18T07:54:33,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742398_1574 (size=12151) 2024-11-18T07:54:33,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/ff8d30b8ea684a699bf2e61303b7ddba 2024-11-18T07:54:33,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/2c875fe1192b4b75932fa35a50a8f411 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2c875fe1192b4b75932fa35a50a8f411 2024-11-18T07:54:33,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2c875fe1192b4b75932fa35a50a8f411, entries=150, sequenceid=218, filesize=30.4 K 2024-11-18T07:54:33,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/a1b9e8d0873c40d983a3a4b9a53f3fa4 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/a1b9e8d0873c40d983a3a4b9a53f3fa4 2024-11-18T07:54:33,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/a1b9e8d0873c40d983a3a4b9a53f3fa4, entries=150, sequenceid=218, filesize=11.9 K 2024-11-18T07:54:33,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/ff8d30b8ea684a699bf2e61303b7ddba as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/ff8d30b8ea684a699bf2e61303b7ddba 2024-11-18T07:54:33,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/ff8d30b8ea684a699bf2e61303b7ddba, entries=150, sequenceid=218, filesize=11.9 K 2024-11-18T07:54:33,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 240ms, sequenceid=218, compaction requested=true 2024-11-18T07:54:33,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:33,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:33,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:33,493 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:33,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:33,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:33,493 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:33,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:33,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:33,494 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93662 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:33,494 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:33,494 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/B is initiating minor compaction (all files) 2024-11-18T07:54:33,494 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/A is initiating minor compaction (all files) 2024-11-18T07:54:33,494 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/B in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:33,494 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/A in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:33,494 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/e261f20ea3644328a4c9c7118a4d32df, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b28109b7d9e04aca82b953f85c9fffac, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/a1b9e8d0873c40d983a3a4b9a53f3fa4] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=35.8 K 2024-11-18T07:54:33,494 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/6e522757facf443e855c8b8e607cd402, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/622ce6960eaf4afa9334c46982e7b331, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2c875fe1192b4b75932fa35a50a8f411] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=91.5 K 2024-11-18T07:54:33,494 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:33,494 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/6e522757facf443e855c8b8e607cd402, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/622ce6960eaf4afa9334c46982e7b331, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2c875fe1192b4b75932fa35a50a8f411] 2024-11-18T07:54:33,494 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting e261f20ea3644328a4c9c7118a4d32df, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731916469532 2024-11-18T07:54:33,495 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e522757facf443e855c8b8e607cd402, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731916469532 2024-11-18T07:54:33,495 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting b28109b7d9e04aca82b953f85c9fffac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731916471910 2024-11-18T07:54:33,495 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 622ce6960eaf4afa9334c46982e7b331, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731916471910 2024-11-18T07:54:33,495 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting a1b9e8d0873c40d983a3a4b9a53f3fa4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731916472615 2024-11-18T07:54:33,495 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c875fe1192b4b75932fa35a50a8f411, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731916472615 2024-11-18T07:54:33,500 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#B#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:33,500 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/a53c9cfac3a64d598e7bf8b911903f65 is 50, key is test_row_0/B:col10/1731916472615/Put/seqid=0 2024-11-18T07:54:33,501 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:33,507 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118c648c34ce9234f219a5f43d55c746191_aea54bdc27847a2fa7b24fdf7cbb20d6 store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:33,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742399_1575 (size=12493) 2024-11-18T07:54:33,508 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118c648c34ce9234f219a5f43d55c746191_aea54bdc27847a2fa7b24fdf7cbb20d6, store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:33,508 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c648c34ce9234f219a5f43d55c746191_aea54bdc27847a2fa7b24fdf7cbb20d6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:33,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742400_1576 (size=4469) 2024-11-18T07:54:33,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916473293, max timestamp=1731916473631Num uniques 859; 2024-11-18T07:54:33,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:33,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916473293, max timestamp=1731916473631Num uniques 859; 2024-11-18T07:54:33,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:33,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=64.85 KB heapSize=184.09 KB 2024-11-18T07:54:33,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916473293, max timestamp=1731916473633Num uniques 859; 2024-11-18T07:54:33,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:33,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:33,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:33,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:33,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:33,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:33,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:33,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186782e044bca94782a8de829528743b5c_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916473634/Put/seqid=0 2024-11-18T07:54:33,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742401_1577 (size=12304) 2024-11-18T07:54:33,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916473686, max timestamp=1731916473696Num uniques 859; 2024-11-18T07:54:33,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:33,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916473686, max timestamp=1731916473697Num uniques 859; 2024-11-18T07:54:33,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:33,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916473686, max timestamp=1731916473697Num uniques 859; 2024-11-18T07:54:33,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:33,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:33,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916533720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:33,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:33,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:33,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916533724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:33,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731916473718, max timestamp=1731916473726Num uniques 1018; 2024-11-18T07:54:33,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-18T07:54:33,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:33,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731916473718, max timestamp=1731916473726Num uniques 1018; 2024-11-18T07:54:33,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-18T07:54:33,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731916473718, max timestamp=1731916473726Num uniques 1018; 2024-11-18T07:54:33,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-18T07:54:33,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731916473747, max timestamp=1731916473768Num uniques 1077; 2024-11-18T07:54:33,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-18T07:54:33,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731916473747, max timestamp=1731916473768Num uniques 1077; 2024-11-18T07:54:33,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731916473747, max timestamp=1731916473768Num uniques 1077; 2024-11-18T07:54:33,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-18T07:54:33,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-18T07:54:33,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916533803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:33,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916533804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:33,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731916473795, max timestamp=1731916473806Num uniques 1086; 2024-11-18T07:54:33,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731916473795, max timestamp=1731916473806Num uniques 1086; 2024-11-18T07:54:33,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-18T07:54:33,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-18T07:54:33,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731916473795, max timestamp=1731916473806Num uniques 1086; 2024-11-18T07:54:33,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-18T07:54:33,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:33,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:33,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731916473861, max timestamp=1731916473880Num uniques 1045; 2024-11-18T07:54:33,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-18T07:54:33,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731916473861, max timestamp=1731916473880Num uniques 1045; 2024-11-18T07:54:33,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-18T07:54:33,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731916473861, max timestamp=1731916473880Num uniques 1045; 2024-11-18T07:54:33,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-18T07:54:33,912 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/a53c9cfac3a64d598e7bf8b911903f65 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/a53c9cfac3a64d598e7bf8b911903f65 2024-11-18T07:54:33,913 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#A#compaction#489 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:33,913 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/724b31c01ddf4b6d8f4fd996834475e1 is 175, key is test_row_0/A:col10/1731916472615/Put/seqid=0 2024-11-18T07:54:33,917 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/B of aea54bdc27847a2fa7b24fdf7cbb20d6 into a53c9cfac3a64d598e7bf8b911903f65(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:33,917 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:33,917 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/B, priority=13, startTime=1731916473493; duration=0sec 2024-11-18T07:54:33,917 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:33,918 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:B 2024-11-18T07:54:33,918 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:33,919 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:33,919 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/C is initiating minor compaction (all files) 2024-11-18T07:54:33,919 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/C in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:33,919 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b579bb4fc91841a6b72a7dd71a0cf248, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/0a506225b9264e30b20813371ec91d4e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/ff8d30b8ea684a699bf2e61303b7ddba] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=35.8 K 2024-11-18T07:54:33,920 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting b579bb4fc91841a6b72a7dd71a0cf248, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731916469532 2024-11-18T07:54:33,920 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a506225b9264e30b20813371ec91d4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731916471910 2024-11-18T07:54:33,920 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting ff8d30b8ea684a699bf2e61303b7ddba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731916472615 2024-11-18T07:54:33,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742402_1578 (size=31447) 2024-11-18T07:54:33,934 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#C#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:33,935 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/0d6e981527fc42d3903dd6875c9b2db8 is 50, key is test_row_0/C:col10/1731916472615/Put/seqid=0 2024-11-18T07:54:33,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731916473922, max timestamp=1731916473932Num uniques 1054; 2024-11-18T07:54:33,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-18T07:54:33,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731916473922, max timestamp=1731916473934Num uniques 1054; 2024-11-18T07:54:33,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-18T07:54:33,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731916473922, max timestamp=1731916473934Num uniques 1054; 2024-11-18T07:54:33,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-18T07:54:33,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742403_1579 (size=12493) 2024-11-18T07:54:33,950 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/0d6e981527fc42d3903dd6875c9b2db8 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/0d6e981527fc42d3903dd6875c9b2db8 2024-11-18T07:54:33,956 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/C of aea54bdc27847a2fa7b24fdf7cbb20d6 into 0d6e981527fc42d3903dd6875c9b2db8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:33,956 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:33,956 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/C, priority=13, startTime=1731916473493; duration=0sec 2024-11-18T07:54:33,956 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:33,956 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:C 2024-11-18T07:54:33,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:33,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916533960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:33,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:33,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916533962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:33,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731916473954, max timestamp=1731916473968Num uniques 1013; 2024-11-18T07:54:33,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-18T07:54:33,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731916473954, max timestamp=1731916473968Num uniques 1013; 2024-11-18T07:54:33,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-18T07:54:33,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:33,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:33,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731916473954, max timestamp=1731916473968Num uniques 1063; 2024-11-18T07:54:33,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-18T07:54:34,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731916474010, max timestamp=1731916474020Num uniques 1072; 2024-11-18T07:54:34,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-18T07:54:34,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731916474010, max timestamp=1731916474020Num uniques 1072; 2024-11-18T07:54:34,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-18T07:54:34,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731916474010, max timestamp=1731916474020Num uniques 1072; 2024-11-18T07:54:34,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-18T07:54:34,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916534059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916534060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916534066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916534078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916534078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,082 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:34,085 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186782e044bca94782a8de829528743b5c_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186782e044bca94782a8de829528743b5c_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:34,085 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/a8f2d53727434dff9dbdf534fb8e3f51, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:34,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/a8f2d53727434dff9dbdf534fb8e3f51 is 175, key is test_row_0/A:col10/1731916473634/Put/seqid=0 2024-11-18T07:54:34,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742404_1580 (size=31105) 2024-11-18T07:54:34,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T07:54:34,168 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:34,169 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:54:34,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-18T07:54:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T07:54:34,170 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:54:34,171 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:54:34,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:34,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916534177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916534178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916534178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T07:54:34,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916534288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916534288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T07:54:34,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:34,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:34,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:34,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:34,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:34,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:34,342 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/724b31c01ddf4b6d8f4fd996834475e1 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/724b31c01ddf4b6d8f4fd996834475e1 2024-11-18T07:54:34,347 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/A of aea54bdc27847a2fa7b24fdf7cbb20d6 into 724b31c01ddf4b6d8f4fd996834475e1(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:34,347 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:34,347 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/A, priority=13, startTime=1731916473493; duration=0sec 2024-11-18T07:54:34,347 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:34,347 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:A 2024-11-18T07:54:34,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916534387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916534388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916534388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T07:54:34,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:34,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:34,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:34,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:34,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:34,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:34,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T07:54:34,491 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/a8f2d53727434dff9dbdf534fb8e3f51 2024-11-18T07:54:34,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/284c544fe8c147f3bd1da14f290604ab is 50, key is test_row_0/B:col10/1731916473634/Put/seqid=0 2024-11-18T07:54:34,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742405_1581 (size=12151) 2024-11-18T07:54:34,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/284c544fe8c147f3bd1da14f290604ab 2024-11-18T07:54:34,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/a74521a6615c412cabfe78099e7b6a06 is 50, key is test_row_0/C:col10/1731916473634/Put/seqid=0 2024-11-18T07:54:34,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742406_1582 (size=12151) 2024-11-18T07:54:34,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/a74521a6615c412cabfe78099e7b6a06 2024-11-18T07:54:34,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916534598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916534598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/a8f2d53727434dff9dbdf534fb8e3f51 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/a8f2d53727434dff9dbdf534fb8e3f51 2024-11-18T07:54:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/a8f2d53727434dff9dbdf534fb8e3f51, entries=150, sequenceid=241, filesize=30.4 K 2024-11-18T07:54:34,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/284c544fe8c147f3bd1da14f290604ab as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/284c544fe8c147f3bd1da14f290604ab 2024-11-18T07:54:34,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/284c544fe8c147f3bd1da14f290604ab, entries=150, sequenceid=241, filesize=11.9 K 2024-11-18T07:54:34,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/a74521a6615c412cabfe78099e7b6a06 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/a74521a6615c412cabfe78099e7b6a06 2024-11-18T07:54:34,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/a74521a6615c412cabfe78099e7b6a06, entries=150, sequenceid=241, filesize=11.9 K 2024-11-18T07:54:34,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=147.60 KB/151140 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 987ms, sequenceid=241, compaction requested=false 2024-11-18T07:54:34,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:34,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T07:54:34,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:34,630 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=147.60 KB heapSize=414.23 KB 2024-11-18T07:54:34,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:34,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:34,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:34,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:34,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:34,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:34,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e352dec840294729b1ed9174abe68fe2_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916474049/Put/seqid=0 2024-11-18T07:54:34,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742407_1583 (size=12454) 2024-11-18T07:54:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:34,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:34,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916534704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916534709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916534709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T07:54:34,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916534818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916534818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916534818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:35,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916535030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:35,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916535030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916535030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:35,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:35,082 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e352dec840294729b1ed9174abe68fe2_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e352dec840294729b1ed9174abe68fe2_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:35,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/d2a8606e71bd4bc99ee294070bd38979, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:35,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/d2a8606e71bd4bc99ee294070bd38979 is 175, key is test_row_0/A:col10/1731916474049/Put/seqid=0 2024-11-18T07:54:35,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742408_1584 (size=31255) 2024-11-18T07:54:35,109 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=401, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/d2a8606e71bd4bc99ee294070bd38979 2024-11-18T07:54:35,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/48af28de71f741d9845bacd4a0c2562b is 50, key is test_row_0/B:col10/1731916474049/Put/seqid=0 2024-11-18T07:54:35,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916535118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:35,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916535118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:35,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742409_1585 (size=12301) 2024-11-18T07:54:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T07:54:35,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916535338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:35,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916535348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:35,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916535358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:35,540 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/48af28de71f741d9845bacd4a0c2562b 2024-11-18T07:54:35,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/47914c7c3bdc4cc59bfb78a16064126d is 50, key is test_row_0/C:col10/1731916474049/Put/seqid=0 2024-11-18T07:54:35,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742410_1586 (size=12301) 2024-11-18T07:54:35,566 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/47914c7c3bdc4cc59bfb78a16064126d 2024-11-18T07:54:35,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/d2a8606e71bd4bc99ee294070bd38979 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/d2a8606e71bd4bc99ee294070bd38979 2024-11-18T07:54:35,582 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/d2a8606e71bd4bc99ee294070bd38979, entries=150, sequenceid=401, filesize=30.5 K 2024-11-18T07:54:35,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/48af28de71f741d9845bacd4a0c2562b as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/48af28de71f741d9845bacd4a0c2562b 2024-11-18T07:54:35,590 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/48af28de71f741d9845bacd4a0c2562b, entries=150, sequenceid=401, filesize=12.0 K 2024-11-18T07:54:35,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/47914c7c3bdc4cc59bfb78a16064126d as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/47914c7c3bdc4cc59bfb78a16064126d 2024-11-18T07:54:35,595 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/47914c7c3bdc4cc59bfb78a16064126d, entries=150, sequenceid=401, filesize=12.0 K 2024-11-18T07:54:35,596 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~414.19 KB/424128, currentSize=33.54 KB/34350 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 966ms, sequenceid=401, compaction requested=true 2024-11-18T07:54:35,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:35,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:35,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-18T07:54:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-18T07:54:35,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-18T07:54:35,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4320 sec 2024-11-18T07:54:35,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4380 sec 2024-11-18T07:54:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:35,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T07:54:35,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:35,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:35,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:35,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:35,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:35,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:35,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111816a87176171c4739af6cd29648266462_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916475850/Put/seqid=0 2024-11-18T07:54:35,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742411_1587 (size=14994) 2024-11-18T07:54:35,884 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:35,887 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111816a87176171c4739af6cd29648266462_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111816a87176171c4739af6cd29648266462_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:35,888 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/ce1a9c90d4fd48dd9badef13b48917af, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:35,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/ce1a9c90d4fd48dd9badef13b48917af is 175, key is test_row_0/A:col10/1731916475850/Put/seqid=0 2024-11-18T07:54:35,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742412_1588 (size=39949) 2024-11-18T07:54:35,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916475877, max timestamp=1731916475942Num uniques 859; 2024-11-18T07:54:35,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916475877, max timestamp=1731916475942Num uniques 859; 2024-11-18T07:54:35,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:35,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:35,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916475877, max timestamp=1731916475942Num uniques 859; 2024-11-18T07:54:35,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:35,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916535981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:35,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916535985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:35,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916535986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:36,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916536104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:36,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916536104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:36,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:36,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916536104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:36,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:36,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:36,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:36,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916536128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:36,131 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_2' in aea54bdc27847a2fa7b24fdf7cbb20d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:54:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:36,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916536138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:36,143 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_1' in aea54bdc27847a2fa7b24fdf7cbb20d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:54:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T07:54:36,319 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:36,320 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:54:36,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916536319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-18T07:54:36,324 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:54:36,325 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:54:36,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T07:54:36,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:36,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916536326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:36,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:36,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:36,327 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=411, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/ce1a9c90d4fd48dd9badef13b48917af 2024-11-18T07:54:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916536326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:36,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/c380a9f8afa449188474b66233c8f390 is 50, key is test_row_0/B:col10/1731916475850/Put/seqid=0 2024-11-18T07:54:36,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742413_1589 (size=12301) 2024-11-18T07:54:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T07:54:36,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T07:54:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:36,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:36,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T07:54:36,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916536628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:36,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:36,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:36,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:36,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:36,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:36,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T07:54:36,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916536641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:36,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916536642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:36,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T07:54:36,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:36,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:36,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:36,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:36,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:36,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/c380a9f8afa449188474b66233c8f390 2024-11-18T07:54:36,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/f336567545334c61a10774ab63d0e121 is 50, key is test_row_0/C:col10/1731916475850/Put/seqid=0 2024-11-18T07:54:36,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742414_1590 (size=12301) 2024-11-18T07:54:36,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T07:54:36,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:36,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:36,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:36,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:36,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:36,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:36,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T07:54:37,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T07:54:37,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:37,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:37,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:37,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:37,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:37,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916537148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:37,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916537158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:37,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916537158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:37,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/f336567545334c61a10774ab63d0e121 2024-11-18T07:54:37,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T07:54:37,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:37,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:37,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:37,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:37,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:37,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/ce1a9c90d4fd48dd9badef13b48917af as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/ce1a9c90d4fd48dd9badef13b48917af 2024-11-18T07:54:37,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/ce1a9c90d4fd48dd9badef13b48917af, entries=200, sequenceid=411, filesize=39.0 K 2024-11-18T07:54:37,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/c380a9f8afa449188474b66233c8f390 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/c380a9f8afa449188474b66233c8f390 2024-11-18T07:54:37,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/c380a9f8afa449188474b66233c8f390, entries=150, sequenceid=411, filesize=12.0 K 2024-11-18T07:54:37,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/f336567545334c61a10774ab63d0e121 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/f336567545334c61a10774ab63d0e121 2024-11-18T07:54:37,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/f336567545334c61a10774ab63d0e121, entries=150, sequenceid=411, filesize=12.0 K 2024-11-18T07:54:37,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 1468ms, sequenceid=411, compaction requested=true 2024-11-18T07:54:37,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:37,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:37,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:37,330 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:54:37,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:37,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:37,330 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:54:37,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:37,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:37,331 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133756 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:54:37,331 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49246 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:54:37,331 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/A is initiating minor compaction (all files) 2024-11-18T07:54:37,331 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/B is initiating minor compaction (all files) 2024-11-18T07:54:37,331 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/B in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:37,331 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/A in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:37,331 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/a53c9cfac3a64d598e7bf8b911903f65, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/284c544fe8c147f3bd1da14f290604ab, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/48af28de71f741d9845bacd4a0c2562b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/c380a9f8afa449188474b66233c8f390] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=48.1 K 2024-11-18T07:54:37,331 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/724b31c01ddf4b6d8f4fd996834475e1, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/a8f2d53727434dff9dbdf534fb8e3f51, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/d2a8606e71bd4bc99ee294070bd38979, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/ce1a9c90d4fd48dd9badef13b48917af] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=130.6 K 2024-11-18T07:54:37,332 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:37,332 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/724b31c01ddf4b6d8f4fd996834475e1, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/a8f2d53727434dff9dbdf534fb8e3f51, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/d2a8606e71bd4bc99ee294070bd38979, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/ce1a9c90d4fd48dd9badef13b48917af] 2024-11-18T07:54:37,332 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 724b31c01ddf4b6d8f4fd996834475e1, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731916472615 2024-11-18T07:54:37,332 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting a53c9cfac3a64d598e7bf8b911903f65, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731916472615 2024-11-18T07:54:37,333 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 284c544fe8c147f3bd1da14f290604ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731916473298 2024-11-18T07:54:37,333 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8f2d53727434dff9dbdf534fb8e3f51, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731916473298 2024-11-18T07:54:37,334 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 48af28de71f741d9845bacd4a0c2562b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731916474049 2024-11-18T07:54:37,334 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2a8606e71bd4bc99ee294070bd38979, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731916474049 2024-11-18T07:54:37,334 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting c380a9f8afa449188474b66233c8f390, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731916474707 2024-11-18T07:54:37,335 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce1a9c90d4fd48dd9badef13b48917af, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731916474707 2024-11-18T07:54:37,345 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#B#compaction#500 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:37,346 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/f7e05aab7dc6467287d2d9dd0aee4f56 is 50, key is test_row_0/B:col10/1731916475850/Put/seqid=0 2024-11-18T07:54:37,355 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:37,383 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411186e176fd6fdd842a985652f3fde42b628_aea54bdc27847a2fa7b24fdf7cbb20d6 store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:37,386 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411186e176fd6fdd842a985652f3fde42b628_aea54bdc27847a2fa7b24fdf7cbb20d6, store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:37,386 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186e176fd6fdd842a985652f3fde42b628_aea54bdc27847a2fa7b24fdf7cbb20d6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:37,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742415_1591 (size=12779) 2024-11-18T07:54:37,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T07:54:37,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:37,415 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-18T07:54:37,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:37,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:37,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:37,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:37,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:37,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:37,419 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/f7e05aab7dc6467287d2d9dd0aee4f56 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/f7e05aab7dc6467287d2d9dd0aee4f56 2024-11-18T07:54:37,424 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/B of aea54bdc27847a2fa7b24fdf7cbb20d6 into f7e05aab7dc6467287d2d9dd0aee4f56(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:37,424 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:37,424 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/B, priority=12, startTime=1731916477330; duration=0sec 2024-11-18T07:54:37,424 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:37,424 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:B 2024-11-18T07:54:37,424 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T07:54:37,426 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49246 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T07:54:37,426 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/C is initiating minor compaction (all files) 2024-11-18T07:54:37,426 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/C in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:37,426 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/0d6e981527fc42d3903dd6875c9b2db8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/a74521a6615c412cabfe78099e7b6a06, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/47914c7c3bdc4cc59bfb78a16064126d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/f336567545334c61a10774ab63d0e121] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=48.1 K 2024-11-18T07:54:37,427 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d6e981527fc42d3903dd6875c9b2db8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731916472615 2024-11-18T07:54:37,430 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting a74521a6615c412cabfe78099e7b6a06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731916473298 2024-11-18T07:54:37,431 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 47914c7c3bdc4cc59bfb78a16064126d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731916474049 2024-11-18T07:54:37,431 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting f336567545334c61a10774ab63d0e121, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731916474707 2024-11-18T07:54:37,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742416_1592 (size=4469) 2024-11-18T07:54:37,438 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#A#compaction#501 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:37,439 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/aa3070e65ee54cefa875d4d396c1b652 is 175, key is test_row_0/A:col10/1731916475850/Put/seqid=0 2024-11-18T07:54:37,460 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#C#compaction#502 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:37,460 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/41fbac8b61584b8883c3a9515b5dbed9 is 50, key is test_row_0/C:col10/1731916475850/Put/seqid=0 2024-11-18T07:54:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T07:54:37,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111866acf67deb25491b92233c62a8f5f932_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916475969/Put/seqid=0 2024-11-18T07:54:37,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742417_1593 (size=31733) 2024-11-18T07:54:37,542 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/aa3070e65ee54cefa875d4d396c1b652 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/aa3070e65ee54cefa875d4d396c1b652 2024-11-18T07:54:37,547 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/A of aea54bdc27847a2fa7b24fdf7cbb20d6 into aa3070e65ee54cefa875d4d396c1b652(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:37,547 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:37,547 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/A, priority=12, startTime=1731916477330; duration=0sec 2024-11-18T07:54:37,547 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:37,547 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:A 2024-11-18T07:54:37,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742418_1594 (size=12779) 2024-11-18T07:54:37,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742419_1595 (size=12454) 2024-11-18T07:54:37,991 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/41fbac8b61584b8883c3a9515b5dbed9 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/41fbac8b61584b8883c3a9515b5dbed9 2024-11-18T07:54:38,005 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/C of aea54bdc27847a2fa7b24fdf7cbb20d6 into 41fbac8b61584b8883c3a9515b5dbed9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:38,005 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:38,005 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/C, priority=12, startTime=1731916477330; duration=0sec 2024-11-18T07:54:38,005 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:38,005 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:C 2024-11-18T07:54:38,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:38,037 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111866acf67deb25491b92233c62a8f5f932_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111866acf67deb25491b92233c62a8f5f932_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:38,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/98f515013633405ea020742e5daa70b8, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:38,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/98f515013633405ea020742e5daa70b8 is 175, key is test_row_0/A:col10/1731916475969/Put/seqid=0 2024-11-18T07:54:38,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742420_1596 (size=31255) 2024-11-18T07:54:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:38,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:38,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916538168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:38,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916538169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:38,174 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_0' in aea54bdc27847a2fa7b24fdf7cbb20d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:54:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916538169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:38,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916538170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:38,175 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_0' in aea54bdc27847a2fa7b24fdf7cbb20d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:54:38,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:38,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:38,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916538179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:38,185 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_1' in aea54bdc27847a2fa7b24fdf7cbb20d6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:54:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:38,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:38,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916538278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:38,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:38,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:38,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916538278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:38,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T07:54:38,490 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=447, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/98f515013633405ea020742e5daa70b8 2024-11-18T07:54:38,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916538488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:38,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916538488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:38,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/9fec1756d0d948658546b7efb3e9aa6f is 50, key is test_row_0/B:col10/1731916475969/Put/seqid=0 2024-11-18T07:54:38,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742421_1597 (size=12301) 2024-11-18T07:54:38,519 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/9fec1756d0d948658546b7efb3e9aa6f 2024-11-18T07:54:38,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/5441f3e5cf5c435ca1ccd0bbf6863ce7 is 50, key is test_row_0/C:col10/1731916475969/Put/seqid=0 2024-11-18T07:54:38,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742422_1598 (size=12301) 2024-11-18T07:54:38,552 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/5441f3e5cf5c435ca1ccd0bbf6863ce7 2024-11-18T07:54:38,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/98f515013633405ea020742e5daa70b8 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/98f515013633405ea020742e5daa70b8 2024-11-18T07:54:38,571 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/98f515013633405ea020742e5daa70b8, entries=150, sequenceid=447, filesize=30.5 K 2024-11-18T07:54:38,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/9fec1756d0d948658546b7efb3e9aa6f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/9fec1756d0d948658546b7efb3e9aa6f 2024-11-18T07:54:38,579 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/9fec1756d0d948658546b7efb3e9aa6f, entries=150, sequenceid=447, filesize=12.0 K 2024-11-18T07:54:38,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/5441f3e5cf5c435ca1ccd0bbf6863ce7 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/5441f3e5cf5c435ca1ccd0bbf6863ce7 2024-11-18T07:54:38,585 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/5441f3e5cf5c435ca1ccd0bbf6863ce7, entries=150, sequenceid=447, filesize=12.0 K 2024-11-18T07:54:38,586 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 1171ms, sequenceid=447, compaction requested=false 2024-11-18T07:54:38,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:38,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:38,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-18T07:54:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-18T07:54:38,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-18T07:54:38,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2620 sec 2024-11-18T07:54:38,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.2700 sec 2024-11-18T07:54:38,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T07:54:38,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:38,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:38,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:38,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:38,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:38,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:38,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111840d0701d6ab84fafaf7bedd5c05de6a0_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916478799/Put/seqid=0 2024-11-18T07:54:38,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742423_1599 (size=17534) 2024-11-18T07:54:38,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731916478843, max timestamp=1731916478856Num uniques 859; 2024-11-18T07:54:38,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:38,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916478843, max timestamp=1731916478856Num uniques 859; 2024-11-18T07:54:38,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:38,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916478843, max timestamp=1731916478856Num uniques 859; 2024-11-18T07:54:38,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:38,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916538902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:38,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916538908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:38,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:38,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:39,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916539019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:39,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916539019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:39,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:39,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:39,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:39,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916539228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:39,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916539228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:39,235 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:39,242 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111840d0701d6ab84fafaf7bedd5c05de6a0_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111840d0701d6ab84fafaf7bedd5c05de6a0_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:39,243 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/c49d0675e3f84b81b845b64285fe8ef0, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:39,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/c49d0675e3f84b81b845b64285fe8ef0 is 175, key is test_row_0/A:col10/1731916478799/Put/seqid=0 2024-11-18T07:54:39,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742424_1600 (size=48639) 2024-11-18T07:54:39,280 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=462, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/c49d0675e3f84b81b845b64285fe8ef0 2024-11-18T07:54:39,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/d28a6da688af4d5991253315ec0f4914 is 50, key is test_row_0/B:col10/1731916478799/Put/seqid=0 2024-11-18T07:54:39,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742425_1601 (size=12301) 2024-11-18T07:54:39,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/d28a6da688af4d5991253315ec0f4914 2024-11-18T07:54:39,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/ee2a457cd9cf4654a09d449ba3f00d5f is 50, key is test_row_0/C:col10/1731916478799/Put/seqid=0 2024-11-18T07:54:39,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742426_1602 (size=12301) 2024-11-18T07:54:39,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916539538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:39,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:39,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:39,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:39,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916539538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:39,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:39,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:39,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:39,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/ee2a457cd9cf4654a09d449ba3f00d5f 2024-11-18T07:54:39,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/c49d0675e3f84b81b845b64285fe8ef0 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/c49d0675e3f84b81b845b64285fe8ef0 2024-11-18T07:54:39,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/c49d0675e3f84b81b845b64285fe8ef0, entries=250, sequenceid=462, filesize=47.5 K 2024-11-18T07:54:39,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/d28a6da688af4d5991253315ec0f4914 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/d28a6da688af4d5991253315ec0f4914 2024-11-18T07:54:39,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/d28a6da688af4d5991253315ec0f4914, entries=150, sequenceid=462, filesize=12.0 K 2024-11-18T07:54:39,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/ee2a457cd9cf4654a09d449ba3f00d5f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/ee2a457cd9cf4654a09d449ba3f00d5f 2024-11-18T07:54:39,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/ee2a457cd9cf4654a09d449ba3f00d5f, entries=150, sequenceid=462, filesize=12.0 K 2024-11-18T07:54:39,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 1020ms, sequenceid=462, compaction requested=true 2024-11-18T07:54:39,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:39,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:39,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:39,820 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:39,820 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:39,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:39,821 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:39,821 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/A is initiating minor compaction (all files) 2024-11-18T07:54:39,821 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:39,821 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/B is initiating minor compaction (all files) 2024-11-18T07:54:39,821 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/A in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:39,821 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/B in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:39,821 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/f7e05aab7dc6467287d2d9dd0aee4f56, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/9fec1756d0d948658546b7efb3e9aa6f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/d28a6da688af4d5991253315ec0f4914] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=36.5 K 2024-11-18T07:54:39,821 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/aa3070e65ee54cefa875d4d396c1b652, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/98f515013633405ea020742e5daa70b8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/c49d0675e3f84b81b845b64285fe8ef0] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=109.0 K 2024-11-18T07:54:39,821 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:39,821 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/aa3070e65ee54cefa875d4d396c1b652, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/98f515013633405ea020742e5daa70b8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/c49d0675e3f84b81b845b64285fe8ef0] 2024-11-18T07:54:39,822 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7e05aab7dc6467287d2d9dd0aee4f56, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731916474707 2024-11-18T07:54:39,822 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting aa3070e65ee54cefa875d4d396c1b652, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731916474707 2024-11-18T07:54:39,822 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fec1756d0d948658546b7efb3e9aa6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731916475969 2024-11-18T07:54:39,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:39,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:39,822 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d28a6da688af4d5991253315ec0f4914, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731916478156 2024-11-18T07:54:39,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:39,822 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 98f515013633405ea020742e5daa70b8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731916475969 2024-11-18T07:54:39,823 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting c49d0675e3f84b81b845b64285fe8ef0, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731916478151 2024-11-18T07:54:39,840 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#B#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:39,841 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/103e64593934413c91821f28070dd70e is 50, key is test_row_0/B:col10/1731916478799/Put/seqid=0 2024-11-18T07:54:39,861 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:39,873 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411187351628443d542068ecf981c23e70e3b_aea54bdc27847a2fa7b24fdf7cbb20d6 store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:39,875 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411187351628443d542068ecf981c23e70e3b_aea54bdc27847a2fa7b24fdf7cbb20d6, store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:39,875 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187351628443d542068ecf981c23e70e3b_aea54bdc27847a2fa7b24fdf7cbb20d6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:39,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742427_1603 (size=12881) 2024-11-18T07:54:39,908 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/103e64593934413c91821f28070dd70e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/103e64593934413c91821f28070dd70e 2024-11-18T07:54:39,912 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/B of aea54bdc27847a2fa7b24fdf7cbb20d6 into 103e64593934413c91821f28070dd70e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:39,913 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:39,913 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/B, priority=13, startTime=1731916479820; duration=0sec 2024-11-18T07:54:39,913 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:39,913 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:B 2024-11-18T07:54:39,913 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:39,925 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:39,925 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/C is initiating minor compaction (all files) 2024-11-18T07:54:39,925 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/C in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:39,925 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/41fbac8b61584b8883c3a9515b5dbed9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/5441f3e5cf5c435ca1ccd0bbf6863ce7, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/ee2a457cd9cf4654a09d449ba3f00d5f] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=36.5 K 2024-11-18T07:54:39,928 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41fbac8b61584b8883c3a9515b5dbed9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731916474707 2024-11-18T07:54:39,928 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5441f3e5cf5c435ca1ccd0bbf6863ce7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731916475969 2024-11-18T07:54:39,931 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee2a457cd9cf4654a09d449ba3f00d5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731916478156 2024-11-18T07:54:39,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742428_1604 (size=4469) 2024-11-18T07:54:39,958 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#A#compaction#510 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:39,959 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#C#compaction#511 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:39,959 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/7fe7384973b5456ca172ce0f8e77ce64 is 175, key is test_row_0/A:col10/1731916478799/Put/seqid=0 2024-11-18T07:54:39,961 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/9e697b57040648af8871c96cdc179d92 is 50, key is test_row_0/C:col10/1731916478799/Put/seqid=0 2024-11-18T07:54:39,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742430_1606 (size=12881) 2024-11-18T07:54:39,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742429_1605 (size=31835) 2024-11-18T07:54:40,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-18T07:54:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:40,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:40,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:40,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:40,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:40,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:40,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:40,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c5ae7049ae0a4006848690569e1135de_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916478901/Put/seqid=0 2024-11-18T07:54:40,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916540102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:40,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916540107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:40,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742431_1607 (size=17534) 2024-11-18T07:54:40,140 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:40,145 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c5ae7049ae0a4006848690569e1135de_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c5ae7049ae0a4006848690569e1135de_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:40,146 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/2d608e475a81433996db4e2a498bf5c4, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:40,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/2d608e475a81433996db4e2a498bf5c4 is 175, key is test_row_0/A:col10/1731916478901/Put/seqid=0 2024-11-18T07:54:40,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742432_1608 (size=48639) 2024-11-18T07:54:40,172 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=499, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/2d608e475a81433996db4e2a498bf5c4 2024-11-18T07:54:40,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916540188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:40,193 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_0' in aea54bdc27847a2fa7b24fdf7cbb20d6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:54:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:40,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/b5b9fec5922541b0a22fe6bb32c338a8 is 50, key is test_row_0/B:col10/1731916478901/Put/seqid=0 2024-11-18T07:54:40,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916540198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:40,206 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_0' in aea54bdc27847a2fa7b24fdf7cbb20d6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:54:40,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:40,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916540201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:40,207 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb3d252950,37987,1731916323128 for 'test_row_1' in aea54bdc27847a2fa7b24fdf7cbb20d6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-18T07:54:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:40,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742433_1609 (size=12301) 2024-11-18T07:54:40,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/b5b9fec5922541b0a22fe6bb32c338a8 2024-11-18T07:54:40,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916540219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:40,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916540228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:40,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/cb77619625af4337b075c38203a1ce89 is 50, key is test_row_0/C:col10/1731916478901/Put/seqid=0 2024-11-18T07:54:40,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742434_1610 (size=12301) 2024-11-18T07:54:40,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/cb77619625af4337b075c38203a1ce89 2024-11-18T07:54:40,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/2d608e475a81433996db4e2a498bf5c4 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2d608e475a81433996db4e2a498bf5c4 2024-11-18T07:54:40,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2d608e475a81433996db4e2a498bf5c4, entries=250, sequenceid=499, filesize=47.5 K 2024-11-18T07:54:40,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/b5b9fec5922541b0a22fe6bb32c338a8 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b5b9fec5922541b0a22fe6bb32c338a8 2024-11-18T07:54:40,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b5b9fec5922541b0a22fe6bb32c338a8, entries=150, sequenceid=499, filesize=12.0 K 2024-11-18T07:54:40,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/cb77619625af4337b075c38203a1ce89 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/cb77619625af4337b075c38203a1ce89 2024-11-18T07:54:40,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/cb77619625af4337b075c38203a1ce89, entries=150, sequenceid=499, filesize=12.0 K 2024-11-18T07:54:40,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 203ms, sequenceid=499, compaction requested=false 2024-11-18T07:54:40,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:40,386 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/9e697b57040648af8871c96cdc179d92 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/9e697b57040648af8871c96cdc179d92 2024-11-18T07:54:40,392 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/C of aea54bdc27847a2fa7b24fdf7cbb20d6 into 9e697b57040648af8871c96cdc179d92(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:40,392 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:40,392 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/C, priority=13, startTime=1731916479822; duration=0sec 2024-11-18T07:54:40,393 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:40,393 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:C 2024-11-18T07:54:40,393 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/7fe7384973b5456ca172ce0f8e77ce64 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/7fe7384973b5456ca172ce0f8e77ce64 2024-11-18T07:54:40,397 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/A of aea54bdc27847a2fa7b24fdf7cbb20d6 into 7fe7384973b5456ca172ce0f8e77ce64(size=31.1 K), total size for store is 78.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:40,397 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:40,397 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/A, priority=13, startTime=1731916479820; duration=0sec 2024-11-18T07:54:40,397 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:40,397 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:A 2024-11-18T07:54:40,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:40,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T07:54:40,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:40,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:40,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:40,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:40,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:40,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:40,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181b3392aa60094f2192d81ed7132ed7e5_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916480097/Put/seqid=0 2024-11-18T07:54:40,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742435_1611 (size=12454) 2024-11-18T07:54:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T07:54:40,498 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:40,498 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:54:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-18T07:54:40,500 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:54:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T07:54:40,500 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:54:40,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:40,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916480470, max timestamp=1731916480508Num uniques 859; 2024-11-18T07:54:40,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916480470, max timestamp=1731916480508Num uniques 859; 2024-11-18T07:54:40,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:40,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:40,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731916480470, max timestamp=1731916480508Num uniques 859; 2024-11-18T07:54:40,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T07:54:40,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916540597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:40,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:40,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T07:54:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916540597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:40,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T07:54:40,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:40,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:40,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:40,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:40,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:40,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916540708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:40,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916540718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:40,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T07:54:40,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:40,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:40,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:40,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:40,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T07:54:40,864 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:40,868 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181b3392aa60094f2192d81ed7132ed7e5_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181b3392aa60094f2192d81ed7132ed7e5_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:40,870 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/f49840f964364d4b8bb394192958c9d1, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:40,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/f49840f964364d4b8bb394192958c9d1 is 175, key is test_row_0/A:col10/1731916480097/Put/seqid=0 2024-11-18T07:54:40,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742436_1612 (size=31255) 2024-11-18T07:54:40,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916540918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:40,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916540928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:40,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T07:54:40,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:40,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:40,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:40,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:40,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T07:54:41,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:41,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:41,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:41,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T07:54:41,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:41,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916541229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:41,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:41,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:41,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:41,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916541238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:41,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:41,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:41,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:41,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T07:54:41,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:41,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:41,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:41,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,318 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=512, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/f49840f964364d4b8bb394192958c9d1 2024-11-18T07:54:41,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/f5de29bb6d254de3a633f5db641f3c0a is 50, key is test_row_0/B:col10/1731916480097/Put/seqid=0 2024-11-18T07:54:41,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742437_1613 (size=12301) 2024-11-18T07:54:41,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T07:54:41,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:41,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:41,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:41,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T07:54:41,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:41,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:41,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:41,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T07:54:41,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/f5de29bb6d254de3a633f5db641f3c0a 2024-11-18T07:54:41,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T07:54:41,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:41,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:41,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:41,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/b2f4b1a9491642cca3fbbd1f87190b33 is 50, key is test_row_0/C:col10/1731916480097/Put/seqid=0 2024-11-18T07:54:41,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916541747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:41,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916541747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:41,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742438_1614 (size=12301) 2024-11-18T07:54:41,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T07:54:41,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:41,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:41,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:41,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:42,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T07:54:42,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:42,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:42,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:42,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:42,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:42,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/b2f4b1a9491642cca3fbbd1f87190b33 2024-11-18T07:54:42,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/f49840f964364d4b8bb394192958c9d1 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/f49840f964364d4b8bb394192958c9d1 2024-11-18T07:54:42,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/f49840f964364d4b8bb394192958c9d1, entries=150, sequenceid=512, filesize=30.5 K 2024-11-18T07:54:42,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/f5de29bb6d254de3a633f5db641f3c0a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/f5de29bb6d254de3a633f5db641f3c0a 2024-11-18T07:54:42,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/f5de29bb6d254de3a633f5db641f3c0a, entries=150, sequenceid=512, filesize=12.0 K 2024-11-18T07:54:42,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/b2f4b1a9491642cca3fbbd1f87190b33 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b2f4b1a9491642cca3fbbd1f87190b33 2024-11-18T07:54:42,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b2f4b1a9491642cca3fbbd1f87190b33, entries=150, sequenceid=512, filesize=12.0 K 2024-11-18T07:54:42,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 1749ms, sequenceid=512, compaction requested=true 2024-11-18T07:54:42,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:42,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:42,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:42,190 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:42,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:42,190 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:42,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:42,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:42,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:42,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T07:54:42,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:42,191 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-18T07:54:42,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:42,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:42,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:42,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:42,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:42,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:42,191 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111729 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:42,191 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:42,191 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/B is initiating minor compaction (all files) 2024-11-18T07:54:42,191 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/A is initiating minor compaction (all files) 2024-11-18T07:54:42,191 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/B in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:42,191 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/A in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:42,191 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/103e64593934413c91821f28070dd70e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b5b9fec5922541b0a22fe6bb32c338a8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/f5de29bb6d254de3a633f5db641f3c0a] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=36.6 K 2024-11-18T07:54:42,191 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/7fe7384973b5456ca172ce0f8e77ce64, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2d608e475a81433996db4e2a498bf5c4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/f49840f964364d4b8bb394192958c9d1] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=109.1 K 2024-11-18T07:54:42,191 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:42,191 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/7fe7384973b5456ca172ce0f8e77ce64, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2d608e475a81433996db4e2a498bf5c4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/f49840f964364d4b8bb394192958c9d1] 2024-11-18T07:54:42,193 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 103e64593934413c91821f28070dd70e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731916478156 2024-11-18T07:54:42,193 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fe7384973b5456ca172ce0f8e77ce64, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731916478156 2024-11-18T07:54:42,194 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting b5b9fec5922541b0a22fe6bb32c338a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731916478901 2024-11-18T07:54:42,194 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d608e475a81433996db4e2a498bf5c4, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731916478873 2024-11-18T07:54:42,194 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting f5de29bb6d254de3a633f5db641f3c0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731916480077 2024-11-18T07:54:42,194 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f49840f964364d4b8bb394192958c9d1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731916480077 2024-11-18T07:54:42,204 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#B#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:42,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183a8f2290771845e38e900301eb002b2e_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916480579/Put/seqid=0 2024-11-18T07:54:42,205 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/ce4598db8b404a939fd30693722cbf27 is 50, key is test_row_0/B:col10/1731916480097/Put/seqid=0 2024-11-18T07:54:42,206 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:42,211 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118670da76f6ea149d99c0b4f78db602005_aea54bdc27847a2fa7b24fdf7cbb20d6 store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:42,213 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118670da76f6ea149d99c0b4f78db602005_aea54bdc27847a2fa7b24fdf7cbb20d6, store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:42,213 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118670da76f6ea149d99c0b4f78db602005_aea54bdc27847a2fa7b24fdf7cbb20d6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:42,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742439_1615 (size=12454) 2024-11-18T07:54:42,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:42,234 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183a8f2290771845e38e900301eb002b2e_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183a8f2290771845e38e900301eb002b2e_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:42,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/f5c9459e1b7f4405aeb501b390aabb63, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:42,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/f5c9459e1b7f4405aeb501b390aabb63 is 175, key is test_row_0/A:col10/1731916480579/Put/seqid=0 2024-11-18T07:54:42,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742440_1616 (size=12983) 2024-11-18T07:54:42,245 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/ce4598db8b404a939fd30693722cbf27 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/ce4598db8b404a939fd30693722cbf27 2024-11-18T07:54:42,250 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/B of aea54bdc27847a2fa7b24fdf7cbb20d6 into ce4598db8b404a939fd30693722cbf27(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:42,250 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:42,250 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/B, priority=13, startTime=1731916482190; duration=0sec 2024-11-18T07:54:42,250 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:42,250 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:B 2024-11-18T07:54:42,251 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:42,252 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:42,252 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/C is initiating minor compaction (all files) 2024-11-18T07:54:42,252 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/C in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:42,254 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/9e697b57040648af8871c96cdc179d92, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/cb77619625af4337b075c38203a1ce89, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b2f4b1a9491642cca3fbbd1f87190b33] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=36.6 K 2024-11-18T07:54:42,255 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e697b57040648af8871c96cdc179d92, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731916478156 2024-11-18T07:54:42,255 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting cb77619625af4337b075c38203a1ce89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731916478901 2024-11-18T07:54:42,255 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting b2f4b1a9491642cca3fbbd1f87190b33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731916480077 2024-11-18T07:54:42,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742441_1617 (size=4469) 2024-11-18T07:54:42,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742442_1618 (size=31255) 2024-11-18T07:54:42,270 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#A#compaction#520 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:42,270 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/53f59bd5ebd04e80b2074dc808d2b05c is 175, key is test_row_0/A:col10/1731916480097/Put/seqid=0 2024-11-18T07:54:42,277 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#C#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:42,278 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/09dc1a0829ac4558ac4c81932d648b58 is 50, key is test_row_0/C:col10/1731916480097/Put/seqid=0 2024-11-18T07:54:42,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742443_1619 (size=31937) 2024-11-18T07:54:42,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742444_1620 (size=12983) 2024-11-18T07:54:42,331 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/53f59bd5ebd04e80b2074dc808d2b05c as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/53f59bd5ebd04e80b2074dc808d2b05c 2024-11-18T07:54:42,337 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/A of aea54bdc27847a2fa7b24fdf7cbb20d6 into 53f59bd5ebd04e80b2074dc808d2b05c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:42,337 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:42,337 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/A, priority=13, startTime=1731916482190; duration=0sec 2024-11-18T07:54:42,337 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:42,337 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:A 2024-11-18T07:54:42,338 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/09dc1a0829ac4558ac4c81932d648b58 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/09dc1a0829ac4558ac4c81932d648b58 2024-11-18T07:54:42,342 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/C of aea54bdc27847a2fa7b24fdf7cbb20d6 into 09dc1a0829ac4558ac4c81932d648b58(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:42,342 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:42,342 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/C, priority=13, startTime=1731916482190; duration=0sec 2024-11-18T07:54:42,342 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:42,342 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:C 2024-11-18T07:54:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T07:54:42,669 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=548, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/f5c9459e1b7f4405aeb501b390aabb63 2024-11-18T07:54:42,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/4722350cfbfb4662809396079bebde2f is 50, key is test_row_0/B:col10/1731916480579/Put/seqid=0 2024-11-18T07:54:42,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742445_1621 (size=12301) 2024-11-18T07:54:42,679 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/4722350cfbfb4662809396079bebde2f 2024-11-18T07:54:42,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/54962228da6b4911a067663a55ee0300 is 50, key is test_row_0/C:col10/1731916480579/Put/seqid=0 2024-11-18T07:54:42,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742446_1622 (size=12301) 2024-11-18T07:54:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:42,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:42,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916542775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:42,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916542780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:42,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:42,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916542888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:42,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916542888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:43,093 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/54962228da6b4911a067663a55ee0300 2024-11-18T07:54:43,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/f5c9459e1b7f4405aeb501b390aabb63 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/f5c9459e1b7f4405aeb501b390aabb63 2024-11-18T07:54:43,100 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/f5c9459e1b7f4405aeb501b390aabb63, entries=150, sequenceid=548, filesize=30.5 K 2024-11-18T07:54:43,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/4722350cfbfb4662809396079bebde2f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/4722350cfbfb4662809396079bebde2f 2024-11-18T07:54:43,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:43,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916543098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:43,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916543098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:43,103 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/4722350cfbfb4662809396079bebde2f, entries=150, sequenceid=548, filesize=12.0 K 2024-11-18T07:54:43,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/54962228da6b4911a067663a55ee0300 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/54962228da6b4911a067663a55ee0300 2024-11-18T07:54:43,106 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/54962228da6b4911a067663a55ee0300, entries=150, sequenceid=548, filesize=12.0 K 2024-11-18T07:54:43,107 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 916ms, sequenceid=548, compaction requested=false 2024-11-18T07:54:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-18T07:54:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-18T07:54:43,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-18T07:54:43,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6080 sec 2024-11-18T07:54:43,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.6120 sec 2024-11-18T07:54:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:43,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T07:54:43,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:43,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:43,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:43,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:43,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:43,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:43,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d1c260b21e444e4f9d0bc2902556f425_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916483409/Put/seqid=0 2024-11-18T07:54:43,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742447_1623 (size=14994) 2024-11-18T07:54:43,421 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:43,423 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d1c260b21e444e4f9d0bc2902556f425_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d1c260b21e444e4f9d0bc2902556f425_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:43,424 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/b9e28614f0b74ed6aaef4ce6a8148f9a, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:43,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/b9e28614f0b74ed6aaef4ce6a8148f9a is 175, key is test_row_0/A:col10/1731916483409/Put/seqid=0 2024-11-18T07:54:43,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742448_1624 (size=39949) 2024-11-18T07:54:43,427 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=563, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/b9e28614f0b74ed6aaef4ce6a8148f9a 2024-11-18T07:54:43,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/82481ae404e14934a2df6c192bba7966 is 50, key is test_row_0/B:col10/1731916483409/Put/seqid=0 2024-11-18T07:54:43,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742449_1625 (size=12301) 2024-11-18T07:54:43,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/82481ae404e14934a2df6c192bba7966 2024-11-18T07:54:43,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/1dc047aee7f1427784f56495c8629dcf is 50, key is test_row_0/C:col10/1731916483409/Put/seqid=0 2024-11-18T07:54:43,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742450_1626 (size=12301) 2024-11-18T07:54:43,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/1dc047aee7f1427784f56495c8629dcf 2024-11-18T07:54:43,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/b9e28614f0b74ed6aaef4ce6a8148f9a as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/b9e28614f0b74ed6aaef4ce6a8148f9a 2024-11-18T07:54:43,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916543464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:43,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916543465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:43,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/b9e28614f0b74ed6aaef4ce6a8148f9a, entries=200, sequenceid=563, filesize=39.0 K 2024-11-18T07:54:43,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/82481ae404e14934a2df6c192bba7966 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/82481ae404e14934a2df6c192bba7966 2024-11-18T07:54:43,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/82481ae404e14934a2df6c192bba7966, entries=150, sequenceid=563, filesize=12.0 K 2024-11-18T07:54:43,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/1dc047aee7f1427784f56495c8629dcf as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/1dc047aee7f1427784f56495c8629dcf 2024-11-18T07:54:43,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/1dc047aee7f1427784f56495c8629dcf, entries=150, sequenceid=563, filesize=12.0 K 2024-11-18T07:54:43,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 77ms, sequenceid=563, compaction requested=true 2024-11-18T07:54:43,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:43,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:43,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:43,488 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:43,488 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:43,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:43,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:43,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:43,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:43,491 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:43,491 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:43,491 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/B is initiating minor compaction (all files) 2024-11-18T07:54:43,491 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/A is initiating minor compaction (all files) 2024-11-18T07:54:43,491 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/A in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:43,491 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/B in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:43,491 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/53f59bd5ebd04e80b2074dc808d2b05c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/f5c9459e1b7f4405aeb501b390aabb63, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/b9e28614f0b74ed6aaef4ce6a8148f9a] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=100.7 K 2024-11-18T07:54:43,491 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/ce4598db8b404a939fd30693722cbf27, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/4722350cfbfb4662809396079bebde2f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/82481ae404e14934a2df6c192bba7966] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=36.7 K 2024-11-18T07:54:43,491 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:43,491 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/53f59bd5ebd04e80b2074dc808d2b05c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/f5c9459e1b7f4405aeb501b390aabb63, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/b9e28614f0b74ed6aaef4ce6a8148f9a] 2024-11-18T07:54:43,492 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting ce4598db8b404a939fd30693722cbf27, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731916480077 2024-11-18T07:54:43,492 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53f59bd5ebd04e80b2074dc808d2b05c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731916480077 2024-11-18T07:54:43,492 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4722350cfbfb4662809396079bebde2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731916480556 2024-11-18T07:54:43,492 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c9459e1b7f4405aeb501b390aabb63, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731916480556 2024-11-18T07:54:43,492 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 82481ae404e14934a2df6c192bba7966, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1731916482773 2024-11-18T07:54:43,492 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9e28614f0b74ed6aaef4ce6a8148f9a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1731916482773 2024-11-18T07:54:43,500 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#B#compaction#527 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:43,501 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/e7656591ca34426b8ccebfbec85e26eb is 50, key is test_row_0/B:col10/1731916483409/Put/seqid=0 2024-11-18T07:54:43,511 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:43,515 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118129b3c6a855a4cfc8623d135815501df_aea54bdc27847a2fa7b24fdf7cbb20d6 store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:43,517 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118129b3c6a855a4cfc8623d135815501df_aea54bdc27847a2fa7b24fdf7cbb20d6, store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:43,517 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118129b3c6a855a4cfc8623d135815501df_aea54bdc27847a2fa7b24fdf7cbb20d6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:43,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742451_1627 (size=13085) 2024-11-18T07:54:43,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742452_1628 (size=4469) 2024-11-18T07:54:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:43,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-18T07:54:43,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916483455, max timestamp=1731916483580Num uniques 859; 2024-11-18T07:54:43,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:43,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916483455, max timestamp=1731916483580Num uniques 859; 2024-11-18T07:54:43,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:43,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916483455, max timestamp=1731916483580Num uniques 859; 2024-11-18T07:54:43,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:43,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:43,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:43,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:43,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:43,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:43,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:43,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182e926752a0854bdd9c4cde4a9e42be4a_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916483580/Put/seqid=0 2024-11-18T07:54:43,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742453_1629 (size=14994) 2024-11-18T07:54:43,613 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:43,619 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182e926752a0854bdd9c4cde4a9e42be4a_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182e926752a0854bdd9c4cde4a9e42be4a_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:43,624 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/1bd761ef9b404f8aa11dd02436efa34f, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:43,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/1bd761ef9b404f8aa11dd02436efa34f is 175, key is test_row_0/A:col10/1731916483580/Put/seqid=0 2024-11-18T07:54:43,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742454_1630 (size=39949) 2024-11-18T07:54:43,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916483635, max timestamp=1731916483650Num uniques 859; 2024-11-18T07:54:43,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:43,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916483635, max timestamp=1731916483650Num uniques 859; 2024-11-18T07:54:43,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:43,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731916483635, max timestamp=1731916483650Num uniques 859; 2024-11-18T07:54:43,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T07:54:43,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916543723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:43,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916543727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:43,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731916483704, max timestamp=1731916483718Num uniques 1018; 2024-11-18T07:54:43,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-18T07:54:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:43,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731916483704, max timestamp=1731916483718Num uniques 1018; 2024-11-18T07:54:43,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-18T07:54:43,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731916483704, max timestamp=1731916483718Num uniques 1018; 2024-11-18T07:54:43,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-18T07:54:43,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731916483871, max timestamp=1731916483899Num uniques 1009; 2024-11-18T07:54:43,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731916483871, max timestamp=1731916483899Num uniques 1009; 2024-11-18T07:54:43,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-11-18T07:54:43,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-11-18T07:54:43,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731916483871, max timestamp=1731916483899Num uniques 1009; 2024-11-18T07:54:43,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-11-18T07:54:43,940 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/e7656591ca34426b8ccebfbec85e26eb as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/e7656591ca34426b8ccebfbec85e26eb 2024-11-18T07:54:43,941 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#A#compaction#528 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:43,941 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/26978f1dc9d44d269a8f0d04376b2a50 is 175, key is test_row_0/A:col10/1731916483409/Put/seqid=0 2024-11-18T07:54:43,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742455_1631 (size=32039) 2024-11-18T07:54:43,947 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/B of aea54bdc27847a2fa7b24fdf7cbb20d6 into e7656591ca34426b8ccebfbec85e26eb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:43,947 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:43,947 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/B, priority=13, startTime=1731916483488; duration=0sec 2024-11-18T07:54:43,947 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:43,947 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:B 2024-11-18T07:54:43,947 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:43,948 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:43,948 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/C is initiating minor compaction (all files) 2024-11-18T07:54:43,948 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/C in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:43,948 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/09dc1a0829ac4558ac4c81932d648b58, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/54962228da6b4911a067663a55ee0300, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/1dc047aee7f1427784f56495c8629dcf] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=36.7 K 2024-11-18T07:54:43,948 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 09dc1a0829ac4558ac4c81932d648b58, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731916480077 2024-11-18T07:54:43,948 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 54962228da6b4911a067663a55ee0300, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731916480556 2024-11-18T07:54:43,949 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dc047aee7f1427784f56495c8629dcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1731916482773 2024-11-18T07:54:43,955 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#C#compaction#530 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:43,955 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/4e193a12add248ec8c02fa768b4f84e0 is 50, key is test_row_0/C:col10/1731916483409/Put/seqid=0 2024-11-18T07:54:43,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742456_1632 (size=13085) 2024-11-18T07:54:43,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731916483935, max timestamp=1731916483962Num uniques 1036; 2024-11-18T07:54:43,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-18T07:54:43,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731916483935, max timestamp=1731916483962Num uniques 1036; 2024-11-18T07:54:43,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-18T07:54:43,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731916483943, max timestamp=1731916483962Num uniques 1036; 2024-11-18T07:54:43,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-18T07:54:44,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731916483962, max timestamp=1731916484020Num uniques 1095; 2024-11-18T07:54:44,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-18T07:54:44,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731916483962, max timestamp=1731916484020Num uniques 1095; 2024-11-18T07:54:44,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731916483962, max timestamp=1731916484020Num uniques 1045; 2024-11-18T07:54:44,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-18T07:54:44,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-18T07:54:44,031 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=585, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/1bd761ef9b404f8aa11dd02436efa34f 2024-11-18T07:54:44,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/b047066664464aa29c646b4d7514a376 is 50, key is test_row_0/B:col10/1731916483580/Put/seqid=0 2024-11-18T07:54:44,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742457_1633 (size=12301) 2024-11-18T07:54:44,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/b047066664464aa29c646b4d7514a376 2024-11-18T07:54:44,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/454e36a4081b432caa5ed4a352034e65 is 50, key is test_row_0/C:col10/1731916483580/Put/seqid=0 2024-11-18T07:54:44,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742458_1634 (size=12301) 2024-11-18T07:54:44,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731916484060, max timestamp=1731916484088Num uniques 1104; 2024-11-18T07:54:44,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-18T07:54:44,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731916484060, max timestamp=1731916484088Num uniques 1054; 2024-11-18T07:54:44,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-18T07:54:44,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731916484060, max timestamp=1731916484088Num uniques 1054; 2024-11-18T07:54:44,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-18T07:54:44,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731916484152, max timestamp=1731916484179Num uniques 1013; 2024-11-18T07:54:44,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731916484152, max timestamp=1731916484179Num uniques 1063; 2024-11-18T07:54:44,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-18T07:54:44,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-18T07:54:44,185 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731916484152, max timestamp=1731916484179Num uniques 1013; 2024-11-18T07:54:44,185 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-18T07:54:44,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731916484213, max timestamp=1731916484231Num uniques 1022; 2024-11-18T07:54:44,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-11-18T07:54:44,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731916484213, max timestamp=1731916484231Num uniques 1022; 2024-11-18T07:54:44,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-11-18T07:54:44,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731916484213, max timestamp=1731916484231Num uniques 1022; 2024-11-18T07:54:44,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-11-18T07:54:44,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916544256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916544257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731916484251, max timestamp=1731916484262Num uniques 1131; 2024-11-18T07:54:44,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=12866, suffix heap size=145592,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=281 2024-11-18T07:54:44,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731916484251, max timestamp=1731916484262Num uniques 1131; 2024-11-18T07:54:44,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=12866, suffix heap size=145592,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=281 2024-11-18T07:54:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731916484251, max timestamp=1731916484262Num uniques 1131; 2024-11-18T07:54:44,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-11-18T07:54:44,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916544333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731916484323, max timestamp=1731916484338Num uniques 1140; 2024-11-18T07:54:44,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=8700, suffix heap size=146528,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=190 2024-11-18T07:54:44,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731916484323, max timestamp=1731916484338Num uniques 1090; 2024-11-18T07:54:44,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-18T07:54:44,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731916484323, max timestamp=1731916484338Num uniques 1140; 2024-11-18T07:54:44,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=8700, suffix heap size=146528,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=190 2024-11-18T07:54:44,356 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/26978f1dc9d44d269a8f0d04376b2a50 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/26978f1dc9d44d269a8f0d04376b2a50 2024-11-18T07:54:44,365 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/A of aea54bdc27847a2fa7b24fdf7cbb20d6 into 26978f1dc9d44d269a8f0d04376b2a50(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:44,366 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:44,366 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/A, priority=13, startTime=1731916483488; duration=0sec 2024-11-18T07:54:44,366 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:44,366 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:A 2024-11-18T07:54:44,373 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/4e193a12add248ec8c02fa768b4f84e0 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/4e193a12add248ec8c02fa768b4f84e0 2024-11-18T07:54:44,378 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/C of aea54bdc27847a2fa7b24fdf7cbb20d6 into 4e193a12add248ec8c02fa768b4f84e0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:44,378 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:44,378 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/C, priority=13, startTime=1731916483488; duration=0sec 2024-11-18T07:54:44,378 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:44,378 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:C 2024-11-18T07:54:44,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731916484401, max timestamp=1731916484411Num uniques 1049; 2024-11-18T07:54:44,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-18T07:54:44,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731916484401, max timestamp=1731916484411Num uniques 1049; 2024-11-18T07:54:44,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-18T07:54:44,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731916484401, max timestamp=1731916484411Num uniques 1049; 2024-11-18T07:54:44,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-18T07:54:44,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/454e36a4081b432caa5ed4a352034e65 2024-11-18T07:54:44,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/1bd761ef9b404f8aa11dd02436efa34f as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/1bd761ef9b404f8aa11dd02436efa34f 2024-11-18T07:54:44,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731916484438, max timestamp=1731916484460Num uniques 1058; 2024-11-18T07:54:44,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731916484438, max timestamp=1731916484460Num uniques 1058; 2024-11-18T07:54:44,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-11-18T07:54:44,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-11-18T07:54:44,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731916484438, max timestamp=1731916484460Num uniques 1058; 2024-11-18T07:54:44,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/87fb3d252950:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-11-18T07:54:44,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/1bd761ef9b404f8aa11dd02436efa34f, entries=200, sequenceid=585, filesize=39.0 K 2024-11-18T07:54:44,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/b047066664464aa29c646b4d7514a376 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b047066664464aa29c646b4d7514a376 2024-11-18T07:54:44,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b047066664464aa29c646b4d7514a376, entries=150, sequenceid=585, filesize=12.0 K 2024-11-18T07:54:44,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/454e36a4081b432caa5ed4a352034e65 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/454e36a4081b432caa5ed4a352034e65 2024-11-18T07:54:44,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/454e36a4081b432caa5ed4a352034e65, entries=150, sequenceid=585, filesize=12.0 K 2024-11-18T07:54:44,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=80.51 KB/82440 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 899ms, sequenceid=585, compaction requested=false 2024-11-18T07:54:44,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:44,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=87.22 KB heapSize=240.94 KB 2024-11-18T07:54:44,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:44,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:44,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:44,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:44,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:44,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T07:54:44,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118cc52674dbbf74062beef3c5f031a591c_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916484480/Put/seqid=0 2024-11-18T07:54:44,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742460_1636 (size=27698) 2024-11-18T07:54:44,512 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:44,515 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118cc52674dbbf74062beef3c5f031a591c_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118cc52674dbbf74062beef3c5f031a591c_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:44,516 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/18b4e1de21e94df3a4c3d7d83d3b6ffa, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:44,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/18b4e1de21e94df3a4c3d7d83d3b6ffa is 175, key is test_row_0/A:col10/1731916484480/Put/seqid=0 2024-11-18T07:54:44,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916544508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916544508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916544515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916544516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916544519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742459_1635 (size=83485) 2024-11-18T07:54:44,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916544628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916544628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916544637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916544638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916544638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T07:54:44,668 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:44,669 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T07:54:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-18T07:54:44,671 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T07:54:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T07:54:44,671 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T07:54:44,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T07:54:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T07:54:44,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T07:54:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:44,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:44,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:44,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916544838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916544838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916544848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916544848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916544848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:44,944 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=806, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/18b4e1de21e94df3a4c3d7d83d3b6ffa 2024-11-18T07:54:44,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/44de3498c40342bfbb7facaab1ac07a6 is 50, key is test_row_0/B:col10/1731916484480/Put/seqid=0 2024-11-18T07:54:44,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T07:54:44,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:44,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:44,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:44,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:44,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:44,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:44,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T07:54:45,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742461_1637 (size=12301) 2024-11-18T07:54:45,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T07:54:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:45,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:45,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916545148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:45,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916545148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:45,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:45,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:45,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:45,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916545158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:45,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916545158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:45,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916545158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:45,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T07:54:45,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:45,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:45,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:45,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:45,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T07:54:45,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=806 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/44de3498c40342bfbb7facaab1ac07a6 2024-11-18T07:54:45,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/e41be28378574f1788465f5ec663394c is 50, key is test_row_0/C:col10/1731916484480/Put/seqid=0 2024-11-18T07:54:45,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742462_1638 (size=12301) 2024-11-18T07:54:45,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T07:54:45,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:45,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:45,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:45,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:45,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:45,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T07:54:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:45,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:45,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731916545658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:45,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:45,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47814 deadline: 1731916545668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:45,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47842 deadline: 1731916545668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47884 deadline: 1731916545668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:45,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T07:54:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47874 deadline: 1731916545678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 2024-11-18T07:54:45,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T07:54:45,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., hostname=87fb3d252950,37987,1731916323128, seqNum=5 because the exception is null or not the one we care about 2024-11-18T07:54:45,693 INFO [Thread-2391 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2391. 2024-11-18T07:54:45,693 DEBUG [Thread-2391 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:45,693 DEBUG [Thread-2391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:45,693 DEBUG [Thread-2391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:45,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:45,696 INFO [Thread-2395 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2395. 2024-11-18T07:54:45,696 DEBUG [Thread-2395 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:45,696 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:45,696 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:45,696 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:45,697 INFO [Thread-2399 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2399. 2024-11-18T07:54:45,697 DEBUG [Thread-2399 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:45,697 DEBUG [Thread-2399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:45,697 DEBUG [Thread-2399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:45,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:45,700 INFO [Thread-2393 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2393. 2024-11-18T07:54:45,700 DEBUG [Thread-2393 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:45,700 DEBUG [Thread-2393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:45,700 DEBUG [Thread-2393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:45,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:45,701 INFO [Thread-2397 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2397. 2024-11-18T07:54:45,701 DEBUG [Thread-2397 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:45,701 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:45,701 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:45,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:45,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T07:54:45,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:45,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:45,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:45,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:45,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T07:54:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T07:54:45,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=806 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/e41be28378574f1788465f5ec663394c 2024-11-18T07:54:45,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/18b4e1de21e94df3a4c3d7d83d3b6ffa as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/18b4e1de21e94df3a4c3d7d83d3b6ffa 2024-11-18T07:54:45,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/18b4e1de21e94df3a4c3d7d83d3b6ffa, entries=450, sequenceid=806, filesize=81.5 K 2024-11-18T07:54:45,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/44de3498c40342bfbb7facaab1ac07a6 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/44de3498c40342bfbb7facaab1ac07a6 2024-11-18T07:54:45,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/44de3498c40342bfbb7facaab1ac07a6, entries=150, sequenceid=806, filesize=12.0 K 2024-11-18T07:54:45,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/e41be28378574f1788465f5ec663394c as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/e41be28378574f1788465f5ec663394c 2024-11-18T07:54:45,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/e41be28378574f1788465f5ec663394c, entries=150, sequenceid=806, filesize=12.0 K 2024-11-18T07:54:45,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~240.89 KB/246672, currentSize=93.93 KB/96180 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 1341ms, sequenceid=806, compaction requested=true 2024-11-18T07:54:45,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:45,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T07:54:45,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:45,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T07:54:45,823 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:45,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:45,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea54bdc27847a2fa7b24fdf7cbb20d6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T07:54:45,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:45,823 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:45,823 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 155473 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:45,823 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:45,824 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/A is initiating minor compaction (all files) 2024-11-18T07:54:45,824 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/B is initiating minor compaction (all files) 2024-11-18T07:54:45,824 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/A in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:45,824 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/B in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:45,824 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/26978f1dc9d44d269a8f0d04376b2a50, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/1bd761ef9b404f8aa11dd02436efa34f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/18b4e1de21e94df3a4c3d7d83d3b6ffa] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=151.8 K 2024-11-18T07:54:45,824 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:45,824 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/e7656591ca34426b8ccebfbec85e26eb, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b047066664464aa29c646b4d7514a376, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/44de3498c40342bfbb7facaab1ac07a6] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=36.8 K 2024-11-18T07:54:45,824 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. files: [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/26978f1dc9d44d269a8f0d04376b2a50, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/1bd761ef9b404f8aa11dd02436efa34f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/18b4e1de21e94df3a4c3d7d83d3b6ffa] 2024-11-18T07:54:45,824 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting e7656591ca34426b8ccebfbec85e26eb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1731916482773 2024-11-18T07:54:45,824 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26978f1dc9d44d269a8f0d04376b2a50, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1731916482773 2024-11-18T07:54:45,824 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting b047066664464aa29c646b4d7514a376, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731916483457 2024-11-18T07:54:45,824 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bd761ef9b404f8aa11dd02436efa34f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731916483457 2024-11-18T07:54:45,824 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 44de3498c40342bfbb7facaab1ac07a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=806, earliestPutTs=1731916484469 2024-11-18T07:54:45,824 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18b4e1de21e94df3a4c3d7d83d3b6ffa, keycount=450, bloomtype=ROW, size=81.5 K, encoding=NONE, compression=NONE, seqNum=806, earliestPutTs=1731916484457 2024-11-18T07:54:45,828 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:45,830 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118ae99960c39ef4fc28f41b77d5218ca73_aea54bdc27847a2fa7b24fdf7cbb20d6 store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:45,830 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#B#compaction#536 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:45,830 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/2d895789230844ce9af59bee9d7d54c6 is 50, key is test_row_0/B:col10/1731916484480/Put/seqid=0 2024-11-18T07:54:45,834 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118ae99960c39ef4fc28f41b77d5218ca73_aea54bdc27847a2fa7b24fdf7cbb20d6, store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:45,834 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ae99960c39ef4fc28f41b77d5218ca73_aea54bdc27847a2fa7b24fdf7cbb20d6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:45,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742463_1639 (size=13187) 2024-11-18T07:54:45,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742464_1640 (size=4469) 2024-11-18T07:54:45,838 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#A#compaction#537 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:45,838 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/5d0a9e61241a4f76b8d67a583ba13e18 is 175, key is test_row_0/A:col10/1731916484480/Put/seqid=0 2024-11-18T07:54:45,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742465_1641 (size=32141) 2024-11-18T07:54:45,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T07:54:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:45,904 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T07:54:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:45,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b2d90214adb340d6876b1a612105ce46_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916484506/Put/seqid=0 2024-11-18T07:54:45,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742466_1642 (size=12454) 2024-11-18T07:54:46,240 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/2d895789230844ce9af59bee9d7d54c6 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/2d895789230844ce9af59bee9d7d54c6 2024-11-18T07:54:46,244 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/5d0a9e61241a4f76b8d67a583ba13e18 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/5d0a9e61241a4f76b8d67a583ba13e18 2024-11-18T07:54:46,244 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/B of aea54bdc27847a2fa7b24fdf7cbb20d6 into 2d895789230844ce9af59bee9d7d54c6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:46,244 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:46,244 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/B, priority=13, startTime=1731916485823; duration=0sec 2024-11-18T07:54:46,244 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T07:54:46,244 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:B 2024-11-18T07:54:46,244 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T07:54:46,245 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T07:54:46,245 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1541): aea54bdc27847a2fa7b24fdf7cbb20d6/C is initiating minor compaction (all files) 2024-11-18T07:54:46,245 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea54bdc27847a2fa7b24fdf7cbb20d6/C in TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:46,245 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/4e193a12add248ec8c02fa768b4f84e0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/454e36a4081b432caa5ed4a352034e65, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/e41be28378574f1788465f5ec663394c] into tmpdir=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp, totalSize=36.8 K 2024-11-18T07:54:46,245 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e193a12add248ec8c02fa768b4f84e0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1731916482773 2024-11-18T07:54:46,246 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting 454e36a4081b432caa5ed4a352034e65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731916483457 2024-11-18T07:54:46,246 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] compactions.Compactor(225): Compacting e41be28378574f1788465f5ec663394c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=806, earliestPutTs=1731916484469 2024-11-18T07:54:46,248 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/A of aea54bdc27847a2fa7b24fdf7cbb20d6 into 5d0a9e61241a4f76b8d67a583ba13e18(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:46,248 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:46,248 INFO [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/A, priority=13, startTime=1731916485823; duration=0sec 2024-11-18T07:54:46,248 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:46,248 DEBUG [RS:0;87fb3d252950:37987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:A 2024-11-18T07:54:46,251 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea54bdc27847a2fa7b24fdf7cbb20d6#C#compaction#539 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T07:54:46,252 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/dd991ba943904dff943acaa047227cf5 is 50, key is test_row_0/C:col10/1731916484480/Put/seqid=0 2024-11-18T07:54:46,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742467_1643 (size=13187) 2024-11-18T07:54:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:46,321 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b2d90214adb340d6876b1a612105ce46_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b2d90214adb340d6876b1a612105ce46_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:46,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/d62e0b84c9424371b2bc42d3e245e4b2, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:46,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/d62e0b84c9424371b2bc42d3e245e4b2 is 175, key is test_row_0/A:col10/1731916484506/Put/seqid=0 2024-11-18T07:54:46,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742468_1644 (size=31255) 2024-11-18T07:54:46,663 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/dd991ba943904dff943acaa047227cf5 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/dd991ba943904dff943acaa047227cf5 2024-11-18T07:54:46,667 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea54bdc27847a2fa7b24fdf7cbb20d6/C of aea54bdc27847a2fa7b24fdf7cbb20d6 into dd991ba943904dff943acaa047227cf5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T07:54:46,667 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:46,667 INFO [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., storeName=aea54bdc27847a2fa7b24fdf7cbb20d6/C, priority=13, startTime=1731916485823; duration=0sec 2024-11-18T07:54:46,667 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T07:54:46,667 DEBUG [RS:0;87fb3d252950:37987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea54bdc27847a2fa7b24fdf7cbb20d6:C 2024-11-18T07:54:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37987 {}] regionserver.HRegion(8855): Flush requested on aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:46,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. as already flushing 2024-11-18T07:54:46,679 INFO [Thread-2380 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2380. 2024-11-18T07:54:46,679 DEBUG [Thread-2380 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:46,679 INFO [Thread-2382 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2382. 2024-11-18T07:54:46,679 DEBUG [Thread-2380 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:46,679 INFO [Thread-2388 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2388. 2024-11-18T07:54:46,679 DEBUG [Thread-2382 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:46,679 DEBUG [Thread-2388 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:46,679 DEBUG [Thread-2382 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:46,679 DEBUG [Thread-2388 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:46,679 DEBUG [Thread-2380 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:46,679 DEBUG [Thread-2382 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:46,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:46,679 DEBUG [Thread-2388 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:46,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:46,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:46,680 INFO [Thread-2384 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2384. 2024-11-18T07:54:46,680 DEBUG [Thread-2384 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:46,680 DEBUG [Thread-2384 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:46,680 DEBUG [Thread-2384 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:46,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:46,700 INFO [Thread-2386 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2386. 2024-11-18T07:54:46,700 DEBUG [Thread-2386 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T07:54:46,700 DEBUG [Thread-2386 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:46,700 DEBUG [Thread-2386 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:46,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:46,725 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=823, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/d62e0b84c9424371b2bc42d3e245e4b2 2024-11-18T07:54:46,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/1675d5d16f5a45ec9903169540cfb32e is 50, key is test_row_0/B:col10/1731916484506/Put/seqid=0 2024-11-18T07:54:46,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742469_1645 (size=12301) 2024-11-18T07:54:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T07:54:47,134 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=823 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/1675d5d16f5a45ec9903169540cfb32e 2024-11-18T07:54:47,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/b776848ab3114af1949f118d6e780e25 is 50, key is test_row_0/C:col10/1731916484506/Put/seqid=0 2024-11-18T07:54:47,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742470_1646 (size=12301) 2024-11-18T07:54:47,545 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=823 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/b776848ab3114af1949f118d6e780e25 2024-11-18T07:54:47,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/d62e0b84c9424371b2bc42d3e245e4b2 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/d62e0b84c9424371b2bc42d3e245e4b2 2024-11-18T07:54:47,551 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/d62e0b84c9424371b2bc42d3e245e4b2, entries=150, sequenceid=823, filesize=30.5 K 2024-11-18T07:54:47,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/1675d5d16f5a45ec9903169540cfb32e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/1675d5d16f5a45ec9903169540cfb32e 2024-11-18T07:54:47,554 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/1675d5d16f5a45ec9903169540cfb32e, entries=150, sequenceid=823, filesize=12.0 K 2024-11-18T07:54:47,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/b776848ab3114af1949f118d6e780e25 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b776848ab3114af1949f118d6e780e25 2024-11-18T07:54:47,557 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b776848ab3114af1949f118d6e780e25, entries=150, sequenceid=823, filesize=12.0 K 2024-11-18T07:54:47,557 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=33.54 KB/34350 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 1653ms, sequenceid=823, compaction requested=false 2024-11-18T07:54:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for aea54bdc27847a2fa7b24fdf7cbb20d6: 2024-11-18T07:54:47,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:47,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb3d252950:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-18T07:54:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-18T07:54:47,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-18T07:54:47,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8880 sec 2024-11-18T07:54:47,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.8920 sec 2024-11-18T07:54:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T07:54:48,839 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:48,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T07:54:48,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-18T07:54:48,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 215 2024-11-18T07:54:48,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 216 2024-11-18T07:54:48,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-18T07:54:48,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-18T07:54:48,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T07:54:48,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2019 2024-11-18T07:54:48,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1994 2024-11-18T07:54:48,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2007 2024-11-18T07:54:48,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2004 2024-11-18T07:54:48,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1975 2024-11-18T07:54:48,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T07:54:48,839 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T07:54:48,839 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T07:54:48,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:48,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:48,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:48,840 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T07:54:48,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T07:54:48,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T07:54:48,843 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916488842"}]},"ts":"1731916488842"} 2024-11-18T07:54:48,844 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T07:54:48,844 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T07:54:48,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T07:54:48,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea54bdc27847a2fa7b24fdf7cbb20d6, UNASSIGN}] 2024-11-18T07:54:48,846 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea54bdc27847a2fa7b24fdf7cbb20d6, UNASSIGN 2024-11-18T07:54:48,847 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=aea54bdc27847a2fa7b24fdf7cbb20d6, regionState=CLOSING, regionLocation=87fb3d252950,37987,1731916323128 2024-11-18T07:54:48,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea54bdc27847a2fa7b24fdf7cbb20d6, UNASSIGN because future has completed 2024-11-18T07:54:48,849 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T07:54:48,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128}] 2024-11-18T07:54:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T07:54:49,001 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:49,001 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T07:54:49,001 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing aea54bdc27847a2fa7b24fdf7cbb20d6, disabling compactions & flushes 2024-11-18T07:54:49,001 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:49,001 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:49,001 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. after waiting 0 ms 2024-11-18T07:54:49,001 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:49,001 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing aea54bdc27847a2fa7b24fdf7cbb20d6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-18T07:54:49,002 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=A 2024-11-18T07:54:49,002 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:49,002 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=B 2024-11-18T07:54:49,002 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:49,002 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea54bdc27847a2fa7b24fdf7cbb20d6, store=C 2024-11-18T07:54:49,002 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T07:54:49,007 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184d576d2df89844698303c4f72db104fe_aea54bdc27847a2fa7b24fdf7cbb20d6 is 50, key is test_row_0/A:col10/1731916486699/Put/seqid=0 2024-11-18T07:54:49,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742471_1647 (size=9914) 2024-11-18T07:54:49,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T07:54:49,411 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T07:54:49,414 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184d576d2df89844698303c4f72db104fe_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184d576d2df89844698303c4f72db104fe_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:49,415 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/7ed652d662af479980a8e970cf8b8e25, store: [table=TestAcidGuarantees family=A region=aea54bdc27847a2fa7b24fdf7cbb20d6] 2024-11-18T07:54:49,416 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/7ed652d662af479980a8e970cf8b8e25 is 175, key is test_row_0/A:col10/1731916486699/Put/seqid=0 2024-11-18T07:54:49,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742472_1648 (size=22561) 2024-11-18T07:54:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T07:54:49,820 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=834, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/7ed652d662af479980a8e970cf8b8e25 2024-11-18T07:54:49,826 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/1b0fc6f909c5489b946e5a2df34c87a6 is 50, key is test_row_0/B:col10/1731916486699/Put/seqid=0 2024-11-18T07:54:49,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742473_1649 (size=9857) 2024-11-18T07:54:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T07:54:50,230 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=834 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/1b0fc6f909c5489b946e5a2df34c87a6 2024-11-18T07:54:50,236 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/033e76c8608d430abd3f195fcc5a9dc0 is 50, key is test_row_0/C:col10/1731916486699/Put/seqid=0 2024-11-18T07:54:50,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742474_1650 (size=9857) 2024-11-18T07:54:50,639 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=834 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/033e76c8608d430abd3f195fcc5a9dc0 2024-11-18T07:54:50,644 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/A/7ed652d662af479980a8e970cf8b8e25 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/7ed652d662af479980a8e970cf8b8e25 2024-11-18T07:54:50,649 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/7ed652d662af479980a8e970cf8b8e25, entries=100, sequenceid=834, filesize=22.0 K 2024-11-18T07:54:50,650 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/B/1b0fc6f909c5489b946e5a2df34c87a6 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/1b0fc6f909c5489b946e5a2df34c87a6 2024-11-18T07:54:50,654 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/1b0fc6f909c5489b946e5a2df34c87a6, entries=100, sequenceid=834, filesize=9.6 K 2024-11-18T07:54:50,655 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/.tmp/C/033e76c8608d430abd3f195fcc5a9dc0 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/033e76c8608d430abd3f195fcc5a9dc0 2024-11-18T07:54:50,660 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/033e76c8608d430abd3f195fcc5a9dc0, entries=100, sequenceid=834, filesize=9.6 K 2024-11-18T07:54:50,661 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 1660ms, sequenceid=834, compaction requested=true 2024-11-18T07:54:50,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2e2cd06a91204d6d9b32bec8700f95ca, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/8700a36212764fd5ab7cf167c5a968c4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/c508216608fd4e29953372b261387b7a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/44705e3f7977439f91449ad8cc66e96b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/b4793be7203b443ab7dc1f0eb27fb433, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/167a80525ab3465290df8a736a060440, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/dfed1b4e06304ccfbee4518cc6e38017, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/6e522757facf443e855c8b8e607cd402, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/622ce6960eaf4afa9334c46982e7b331, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/724b31c01ddf4b6d8f4fd996834475e1, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2c875fe1192b4b75932fa35a50a8f411, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/a8f2d53727434dff9dbdf534fb8e3f51, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/d2a8606e71bd4bc99ee294070bd38979, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/ce1a9c90d4fd48dd9badef13b48917af, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/aa3070e65ee54cefa875d4d396c1b652, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/98f515013633405ea020742e5daa70b8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/c49d0675e3f84b81b845b64285fe8ef0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/7fe7384973b5456ca172ce0f8e77ce64, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2d608e475a81433996db4e2a498bf5c4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/53f59bd5ebd04e80b2074dc808d2b05c, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/f49840f964364d4b8bb394192958c9d1, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/f5c9459e1b7f4405aeb501b390aabb63, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/b9e28614f0b74ed6aaef4ce6a8148f9a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/26978f1dc9d44d269a8f0d04376b2a50, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/1bd761ef9b404f8aa11dd02436efa34f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/18b4e1de21e94df3a4c3d7d83d3b6ffa] to archive 2024-11-18T07:54:50,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:54:50,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2e2cd06a91204d6d9b32bec8700f95ca to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2e2cd06a91204d6d9b32bec8700f95ca 2024-11-18T07:54:50,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/8700a36212764fd5ab7cf167c5a968c4 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/8700a36212764fd5ab7cf167c5a968c4 2024-11-18T07:54:50,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/c508216608fd4e29953372b261387b7a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/c508216608fd4e29953372b261387b7a 2024-11-18T07:54:50,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/44705e3f7977439f91449ad8cc66e96b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/44705e3f7977439f91449ad8cc66e96b 2024-11-18T07:54:50,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/b4793be7203b443ab7dc1f0eb27fb433 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/b4793be7203b443ab7dc1f0eb27fb433 2024-11-18T07:54:50,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/167a80525ab3465290df8a736a060440 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/167a80525ab3465290df8a736a060440 2024-11-18T07:54:50,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/dfed1b4e06304ccfbee4518cc6e38017 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/dfed1b4e06304ccfbee4518cc6e38017 2024-11-18T07:54:50,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/6e522757facf443e855c8b8e607cd402 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/6e522757facf443e855c8b8e607cd402 2024-11-18T07:54:50,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/622ce6960eaf4afa9334c46982e7b331 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/622ce6960eaf4afa9334c46982e7b331 2024-11-18T07:54:50,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/724b31c01ddf4b6d8f4fd996834475e1 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/724b31c01ddf4b6d8f4fd996834475e1 2024-11-18T07:54:50,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2c875fe1192b4b75932fa35a50a8f411 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2c875fe1192b4b75932fa35a50a8f411 2024-11-18T07:54:50,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/a8f2d53727434dff9dbdf534fb8e3f51 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/a8f2d53727434dff9dbdf534fb8e3f51 2024-11-18T07:54:50,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/d2a8606e71bd4bc99ee294070bd38979 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/d2a8606e71bd4bc99ee294070bd38979 2024-11-18T07:54:50,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/ce1a9c90d4fd48dd9badef13b48917af to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/ce1a9c90d4fd48dd9badef13b48917af 2024-11-18T07:54:50,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/aa3070e65ee54cefa875d4d396c1b652 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/aa3070e65ee54cefa875d4d396c1b652 2024-11-18T07:54:50,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/98f515013633405ea020742e5daa70b8 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/98f515013633405ea020742e5daa70b8 2024-11-18T07:54:50,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/c49d0675e3f84b81b845b64285fe8ef0 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/c49d0675e3f84b81b845b64285fe8ef0 2024-11-18T07:54:50,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/7fe7384973b5456ca172ce0f8e77ce64 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/7fe7384973b5456ca172ce0f8e77ce64 2024-11-18T07:54:50,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2d608e475a81433996db4e2a498bf5c4 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/2d608e475a81433996db4e2a498bf5c4 2024-11-18T07:54:50,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/53f59bd5ebd04e80b2074dc808d2b05c to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/53f59bd5ebd04e80b2074dc808d2b05c 2024-11-18T07:54:50,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/f49840f964364d4b8bb394192958c9d1 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/f49840f964364d4b8bb394192958c9d1 2024-11-18T07:54:50,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/f5c9459e1b7f4405aeb501b390aabb63 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/f5c9459e1b7f4405aeb501b390aabb63 2024-11-18T07:54:50,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/b9e28614f0b74ed6aaef4ce6a8148f9a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/b9e28614f0b74ed6aaef4ce6a8148f9a 2024-11-18T07:54:50,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/26978f1dc9d44d269a8f0d04376b2a50 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/26978f1dc9d44d269a8f0d04376b2a50 2024-11-18T07:54:50,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/1bd761ef9b404f8aa11dd02436efa34f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/1bd761ef9b404f8aa11dd02436efa34f 2024-11-18T07:54:50,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/18b4e1de21e94df3a4c3d7d83d3b6ffa to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/18b4e1de21e94df3a4c3d7d83d3b6ffa 2024-11-18T07:54:50,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/6ae2c6b8f79e413bbe7c4313b1765bd4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8e984a70571d4c9ea453867bbf4f6e3f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8490c4a4966b443586c0c7217eecc246, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/0594a3b6c76f4ce1855aba42e64c1076, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/55a134e775684fa9875d6d0f81c6167b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8649c88c851b4694b1371e13bd8115a7, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/e261f20ea3644328a4c9c7118a4d32df, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/82cad5ed754448cdbfaed210c329f0fa, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b28109b7d9e04aca82b953f85c9fffac, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/a53c9cfac3a64d598e7bf8b911903f65, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/a1b9e8d0873c40d983a3a4b9a53f3fa4, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/284c544fe8c147f3bd1da14f290604ab, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/48af28de71f741d9845bacd4a0c2562b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/f7e05aab7dc6467287d2d9dd0aee4f56, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/c380a9f8afa449188474b66233c8f390, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/9fec1756d0d948658546b7efb3e9aa6f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/103e64593934413c91821f28070dd70e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/d28a6da688af4d5991253315ec0f4914, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b5b9fec5922541b0a22fe6bb32c338a8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/ce4598db8b404a939fd30693722cbf27, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/f5de29bb6d254de3a633f5db641f3c0a, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/4722350cfbfb4662809396079bebde2f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/e7656591ca34426b8ccebfbec85e26eb, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/82481ae404e14934a2df6c192bba7966, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b047066664464aa29c646b4d7514a376, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/44de3498c40342bfbb7facaab1ac07a6] to archive 2024-11-18T07:54:50,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:54:50,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/6ae2c6b8f79e413bbe7c4313b1765bd4 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/6ae2c6b8f79e413bbe7c4313b1765bd4 2024-11-18T07:54:50,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8e984a70571d4c9ea453867bbf4f6e3f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8e984a70571d4c9ea453867bbf4f6e3f 2024-11-18T07:54:50,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8490c4a4966b443586c0c7217eecc246 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8490c4a4966b443586c0c7217eecc246 2024-11-18T07:54:50,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/0594a3b6c76f4ce1855aba42e64c1076 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/0594a3b6c76f4ce1855aba42e64c1076 2024-11-18T07:54:50,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/55a134e775684fa9875d6d0f81c6167b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/55a134e775684fa9875d6d0f81c6167b 2024-11-18T07:54:50,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8649c88c851b4694b1371e13bd8115a7 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/8649c88c851b4694b1371e13bd8115a7 2024-11-18T07:54:50,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/e261f20ea3644328a4c9c7118a4d32df to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/e261f20ea3644328a4c9c7118a4d32df 2024-11-18T07:54:50,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/82cad5ed754448cdbfaed210c329f0fa to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/82cad5ed754448cdbfaed210c329f0fa 2024-11-18T07:54:50,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b28109b7d9e04aca82b953f85c9fffac to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b28109b7d9e04aca82b953f85c9fffac 2024-11-18T07:54:50,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/a53c9cfac3a64d598e7bf8b911903f65 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/a53c9cfac3a64d598e7bf8b911903f65 2024-11-18T07:54:50,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/a1b9e8d0873c40d983a3a4b9a53f3fa4 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/a1b9e8d0873c40d983a3a4b9a53f3fa4 2024-11-18T07:54:50,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/284c544fe8c147f3bd1da14f290604ab to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/284c544fe8c147f3bd1da14f290604ab 2024-11-18T07:54:50,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/48af28de71f741d9845bacd4a0c2562b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/48af28de71f741d9845bacd4a0c2562b 2024-11-18T07:54:50,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/f7e05aab7dc6467287d2d9dd0aee4f56 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/f7e05aab7dc6467287d2d9dd0aee4f56 2024-11-18T07:54:50,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/c380a9f8afa449188474b66233c8f390 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/c380a9f8afa449188474b66233c8f390 2024-11-18T07:54:50,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/9fec1756d0d948658546b7efb3e9aa6f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/9fec1756d0d948658546b7efb3e9aa6f 2024-11-18T07:54:50,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/103e64593934413c91821f28070dd70e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/103e64593934413c91821f28070dd70e 2024-11-18T07:54:50,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/d28a6da688af4d5991253315ec0f4914 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/d28a6da688af4d5991253315ec0f4914 2024-11-18T07:54:50,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b5b9fec5922541b0a22fe6bb32c338a8 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b5b9fec5922541b0a22fe6bb32c338a8 2024-11-18T07:54:50,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/ce4598db8b404a939fd30693722cbf27 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/ce4598db8b404a939fd30693722cbf27 2024-11-18T07:54:50,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/f5de29bb6d254de3a633f5db641f3c0a to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/f5de29bb6d254de3a633f5db641f3c0a 2024-11-18T07:54:50,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/4722350cfbfb4662809396079bebde2f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/4722350cfbfb4662809396079bebde2f 2024-11-18T07:54:50,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/e7656591ca34426b8ccebfbec85e26eb to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/e7656591ca34426b8ccebfbec85e26eb 2024-11-18T07:54:50,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/82481ae404e14934a2df6c192bba7966 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/82481ae404e14934a2df6c192bba7966 2024-11-18T07:54:50,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b047066664464aa29c646b4d7514a376 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/b047066664464aa29c646b4d7514a376 2024-11-18T07:54:50,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/44de3498c40342bfbb7facaab1ac07a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/44de3498c40342bfbb7facaab1ac07a6 2024-11-18T07:54:50,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/d892a5dd86e2400694613168fe11a6fc, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/05d876d84187410fa0d4288ccc7163fe, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/c878340e11414e0b8a54c94204bedc95, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/dd4081404b54431fb8196547920df492, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/3924ea7bffe740a8af25c5df5dc53e25, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/e7bfbc09a2ab4a59a8172d887cc9b2d5, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b579bb4fc91841a6b72a7dd71a0cf248, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b294e05fd531467583cb338128eb1e2b, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/0a506225b9264e30b20813371ec91d4e, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/0d6e981527fc42d3903dd6875c9b2db8, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/ff8d30b8ea684a699bf2e61303b7ddba, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/a74521a6615c412cabfe78099e7b6a06, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/47914c7c3bdc4cc59bfb78a16064126d, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/41fbac8b61584b8883c3a9515b5dbed9, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/f336567545334c61a10774ab63d0e121, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/5441f3e5cf5c435ca1ccd0bbf6863ce7, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/9e697b57040648af8871c96cdc179d92, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/ee2a457cd9cf4654a09d449ba3f00d5f, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/cb77619625af4337b075c38203a1ce89, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/09dc1a0829ac4558ac4c81932d648b58, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b2f4b1a9491642cca3fbbd1f87190b33, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/54962228da6b4911a067663a55ee0300, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/4e193a12add248ec8c02fa768b4f84e0, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/1dc047aee7f1427784f56495c8629dcf, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/454e36a4081b432caa5ed4a352034e65, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/e41be28378574f1788465f5ec663394c] to archive 2024-11-18T07:54:50,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T07:54:50,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/d892a5dd86e2400694613168fe11a6fc to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/d892a5dd86e2400694613168fe11a6fc 2024-11-18T07:54:50,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/05d876d84187410fa0d4288ccc7163fe to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/05d876d84187410fa0d4288ccc7163fe 2024-11-18T07:54:50,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/c878340e11414e0b8a54c94204bedc95 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/c878340e11414e0b8a54c94204bedc95 2024-11-18T07:54:50,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/dd4081404b54431fb8196547920df492 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/dd4081404b54431fb8196547920df492 2024-11-18T07:54:50,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/3924ea7bffe740a8af25c5df5dc53e25 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/3924ea7bffe740a8af25c5df5dc53e25 2024-11-18T07:54:50,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/e7bfbc09a2ab4a59a8172d887cc9b2d5 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/e7bfbc09a2ab4a59a8172d887cc9b2d5 2024-11-18T07:54:50,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b579bb4fc91841a6b72a7dd71a0cf248 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b579bb4fc91841a6b72a7dd71a0cf248 2024-11-18T07:54:50,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b294e05fd531467583cb338128eb1e2b to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b294e05fd531467583cb338128eb1e2b 2024-11-18T07:54:50,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/0a506225b9264e30b20813371ec91d4e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/0a506225b9264e30b20813371ec91d4e 2024-11-18T07:54:50,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/0d6e981527fc42d3903dd6875c9b2db8 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/0d6e981527fc42d3903dd6875c9b2db8 2024-11-18T07:54:50,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/ff8d30b8ea684a699bf2e61303b7ddba to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/ff8d30b8ea684a699bf2e61303b7ddba 2024-11-18T07:54:50,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/a74521a6615c412cabfe78099e7b6a06 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/a74521a6615c412cabfe78099e7b6a06 2024-11-18T07:54:50,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/47914c7c3bdc4cc59bfb78a16064126d to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/47914c7c3bdc4cc59bfb78a16064126d 2024-11-18T07:54:50,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/41fbac8b61584b8883c3a9515b5dbed9 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/41fbac8b61584b8883c3a9515b5dbed9 2024-11-18T07:54:50,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/f336567545334c61a10774ab63d0e121 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/f336567545334c61a10774ab63d0e121 2024-11-18T07:54:50,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/5441f3e5cf5c435ca1ccd0bbf6863ce7 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/5441f3e5cf5c435ca1ccd0bbf6863ce7 2024-11-18T07:54:50,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/9e697b57040648af8871c96cdc179d92 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/9e697b57040648af8871c96cdc179d92 2024-11-18T07:54:50,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/ee2a457cd9cf4654a09d449ba3f00d5f to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/ee2a457cd9cf4654a09d449ba3f00d5f 2024-11-18T07:54:50,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/cb77619625af4337b075c38203a1ce89 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/cb77619625af4337b075c38203a1ce89 2024-11-18T07:54:50,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/09dc1a0829ac4558ac4c81932d648b58 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/09dc1a0829ac4558ac4c81932d648b58 2024-11-18T07:54:50,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b2f4b1a9491642cca3fbbd1f87190b33 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b2f4b1a9491642cca3fbbd1f87190b33 2024-11-18T07:54:50,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/54962228da6b4911a067663a55ee0300 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/54962228da6b4911a067663a55ee0300 2024-11-18T07:54:50,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/4e193a12add248ec8c02fa768b4f84e0 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/4e193a12add248ec8c02fa768b4f84e0 2024-11-18T07:54:50,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/1dc047aee7f1427784f56495c8629dcf to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/1dc047aee7f1427784f56495c8629dcf 2024-11-18T07:54:50,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/454e36a4081b432caa5ed4a352034e65 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/454e36a4081b432caa5ed4a352034e65 2024-11-18T07:54:50,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/e41be28378574f1788465f5ec663394c to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/e41be28378574f1788465f5ec663394c 2024-11-18T07:54:50,779 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/recovered.edits/837.seqid, newMaxSeqId=837, maxSeqId=4 2024-11-18T07:54:50,780 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. 2024-11-18T07:54:50,780 DEBUG [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for aea54bdc27847a2fa7b24fdf7cbb20d6: Waiting for close lock at 1731916489001Running coprocessor pre-close hooks at 1731916489001Disabling compacts and flushes for region at 1731916489001Disabling writes for close at 1731916489001Obtaining lock to block concurrent updates at 1731916489001Preparing flush snapshotting stores in aea54bdc27847a2fa7b24fdf7cbb20d6 at 1731916489001Finished memstore snapshotting TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731916489002 (+1 ms)Flushing stores of TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6. at 1731916489002Flushing aea54bdc27847a2fa7b24fdf7cbb20d6/A: creating writer at 1731916489002Flushing mob file aea54bdc27847a2fa7b24fdf7cbb20d6/A: appending metadata at 1731916489007 (+5 ms)Flushing mob file aea54bdc27847a2fa7b24fdf7cbb20d6/A: closing flushed file at 1731916489007Flushing aea54bdc27847a2fa7b24fdf7cbb20d6/A: appending metadata at 1731916489415 (+408 ms)Flushing aea54bdc27847a2fa7b24fdf7cbb20d6/A: closing flushed file at 1731916489415Flushing aea54bdc27847a2fa7b24fdf7cbb20d6/B: creating writer at 1731916489823 (+408 ms)Flushing aea54bdc27847a2fa7b24fdf7cbb20d6/B: appending metadata at 1731916489826 (+3 ms)Flushing aea54bdc27847a2fa7b24fdf7cbb20d6/B: closing flushed file at 1731916489826Flushing aea54bdc27847a2fa7b24fdf7cbb20d6/C: creating writer at 1731916490234 (+408 ms)Flushing aea54bdc27847a2fa7b24fdf7cbb20d6/C: appending metadata at 1731916490236 (+2 ms)Flushing aea54bdc27847a2fa7b24fdf7cbb20d6/C: closing flushed file at 1731916490236Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@86fbe54: reopening flushed file at 1731916490643 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f9d2756: reopening flushed file at 1731916490649 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ca46547: reopening flushed file at 1731916490654 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for aea54bdc27847a2fa7b24fdf7cbb20d6 in 1660ms, sequenceid=834, compaction requested=true at 1731916490661 (+7 ms)Writing region close event to WAL at 1731916490777 (+116 ms)Running coprocessor post-close hooks at 1731916490780 (+3 ms)Closed at 1731916490780 2024-11-18T07:54:50,782 INFO [RS_CLOSE_REGION-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:50,783 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=aea54bdc27847a2fa7b24fdf7cbb20d6, regionState=CLOSED 2024-11-18T07:54:50,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 because future has completed 2024-11-18T07:54:50,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-18T07:54:50,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure aea54bdc27847a2fa7b24fdf7cbb20d6, server=87fb3d252950,37987,1731916323128 in 1.9360 sec 2024-11-18T07:54:50,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-18T07:54:50,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea54bdc27847a2fa7b24fdf7cbb20d6, UNASSIGN in 1.9410 sec 2024-11-18T07:54:50,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-18T07:54:50,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9440 sec 2024-11-18T07:54:50,791 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731916490791"}]},"ts":"1731916490791"} 2024-11-18T07:54:50,793 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T07:54:50,793 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T07:54:50,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9530 sec 2024-11-18T07:54:50,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T07:54:50,988 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:50,988 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T07:54:50,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:54:50,990 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:54:50,991 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:54:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T07:54:51,015 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,036 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C, FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/recovered.edits] 2024-11-18T07:54:51,038 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/5d0a9e61241a4f76b8d67a583ba13e18 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/5d0a9e61241a4f76b8d67a583ba13e18 2024-11-18T07:54:51,039 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/7ed652d662af479980a8e970cf8b8e25 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/7ed652d662af479980a8e970cf8b8e25 2024-11-18T07:54:51,039 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/d62e0b84c9424371b2bc42d3e245e4b2 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/A/d62e0b84c9424371b2bc42d3e245e4b2 2024-11-18T07:54:51,041 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/1675d5d16f5a45ec9903169540cfb32e to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/1675d5d16f5a45ec9903169540cfb32e 2024-11-18T07:54:51,042 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/1b0fc6f909c5489b946e5a2df34c87a6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/1b0fc6f909c5489b946e5a2df34c87a6 2024-11-18T07:54:51,042 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/2d895789230844ce9af59bee9d7d54c6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/B/2d895789230844ce9af59bee9d7d54c6 2024-11-18T07:54:51,049 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/033e76c8608d430abd3f195fcc5a9dc0 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/033e76c8608d430abd3f195fcc5a9dc0 2024-11-18T07:54:51,052 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b776848ab3114af1949f118d6e780e25 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/b776848ab3114af1949f118d6e780e25 2024-11-18T07:54:51,053 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/dd991ba943904dff943acaa047227cf5 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/C/dd991ba943904dff943acaa047227cf5 2024-11-18T07:54:51,055 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/recovered.edits/837.seqid to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6/recovered.edits/837.seqid 2024-11-18T07:54:51,055 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/default/TestAcidGuarantees/aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,055 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T07:54:51,055 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T07:54:51,056 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T07:54:51,058 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118024f953178ac4aae9f42db596d40f55c_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118024f953178ac4aae9f42db596d40f55c_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,059 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180a6a79a3b43a40e08b7183ffe2ae87e3_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180a6a79a3b43a40e08b7183ffe2ae87e3_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,060 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180ff1ed5e928d4b0d8d1ccffec7a58165_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180ff1ed5e928d4b0d8d1ccffec7a58165_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,061 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111816a87176171c4739af6cd29648266462_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111816a87176171c4739af6cd29648266462_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,062 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181b3392aa60094f2192d81ed7132ed7e5_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181b3392aa60094f2192d81ed7132ed7e5_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,063 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182e926752a0854bdd9c4cde4a9e42be4a_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182e926752a0854bdd9c4cde4a9e42be4a_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,064 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182eeedb401c504394bdc5e0869ee7a875_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182eeedb401c504394bdc5e0869ee7a875_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,065 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183a8f2290771845e38e900301eb002b2e_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183a8f2290771845e38e900301eb002b2e_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,066 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111840d0701d6ab84fafaf7bedd5c05de6a0_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111840d0701d6ab84fafaf7bedd5c05de6a0_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,067 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184d576d2df89844698303c4f72db104fe_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184d576d2df89844698303c4f72db104fe_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,067 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118597e0457f88e4bc3a1dfc919f00247b1_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118597e0457f88e4bc3a1dfc919f00247b1_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,068 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111866acf67deb25491b92233c62a8f5f932_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111866acf67deb25491b92233c62a8f5f932_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,069 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111866ef979fe78243d7af2263135e86de30_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111866ef979fe78243d7af2263135e86de30_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,070 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186782e044bca94782a8de829528743b5c_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186782e044bca94782a8de829528743b5c_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,070 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b2d90214adb340d6876b1a612105ce46_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b2d90214adb340d6876b1a612105ce46_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,071 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c5ae7049ae0a4006848690569e1135de_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c5ae7049ae0a4006848690569e1135de_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,072 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118cc52674dbbf74062beef3c5f031a591c_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118cc52674dbbf74062beef3c5f031a591c_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,073 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118cff1e57fd0c640fd896b203b33fbbb9f_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118cff1e57fd0c640fd896b203b33fbbb9f_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,074 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d1c260b21e444e4f9d0bc2902556f425_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d1c260b21e444e4f9d0bc2902556f425_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,075 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e352dec840294729b1ed9174abe68fe2_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e352dec840294729b1ed9174abe68fe2_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,076 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f4628cc6241d47939cf98e8d37404705_aea54bdc27847a2fa7b24fdf7cbb20d6 to hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f4628cc6241d47939cf98e8d37404705_aea54bdc27847a2fa7b24fdf7cbb20d6 2024-11-18T07:54:51,077 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T07:54:51,079 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:54:51,081 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T07:54:51,083 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T07:54:51,084 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:54:51,084 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T07:54:51,084 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731916491084"}]},"ts":"9223372036854775807"} 2024-11-18T07:54:51,086 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T07:54:51,086 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => aea54bdc27847a2fa7b24fdf7cbb20d6, NAME => 'TestAcidGuarantees,,1731916462485.aea54bdc27847a2fa7b24fdf7cbb20d6.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T07:54:51,086 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T07:54:51,086 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731916491086"}]},"ts":"9223372036854775807"} 2024-11-18T07:54:51,088 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T07:54:51,089 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T07:54:51,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 101 msec 2024-11-18T07:54:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T07:54:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T07:54:51,099 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T07:54:51,108 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=253 (was 250) - Thread LEAK? -, OpenFileDescriptor=457 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=938 (was 1010), ProcessCount=11 (was 11), AvailableMemoryMB=2218 (was 1794) - AvailableMemoryMB LEAK? - 2024-11-18T07:54:51,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T07:54:51,108 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T07:54:51,109 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T07:54:51,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:51,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:51,109 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T07:54:51,109 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T07:54:51,109 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1219616701, stopped=false 2024-11-18T07:54:51,109 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=87fb3d252950,42633,1731916322278 2024-11-18T07:54:51,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T07:54:51,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T07:54:51,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T07:54:51,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T07:54:51,169 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T07:54:51,175 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T07:54:51,175 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T07:54:51,175 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T07:54:51,175 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T07:54:51,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:51,176 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '87fb3d252950,37987,1731916323128' ***** 2024-11-18T07:54:51,176 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T07:54:51,176 INFO [RS:0;87fb3d252950:37987 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T07:54:51,177 INFO [RS:0;87fb3d252950:37987 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T07:54:51,177 INFO [RS:0;87fb3d252950:37987 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T07:54:51,177 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T07:54:51,177 INFO [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(959): stopping server 87fb3d252950,37987,1731916323128 2024-11-18T07:54:51,177 INFO [RS:0;87fb3d252950:37987 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T07:54:51,177 INFO [RS:0;87fb3d252950:37987 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;87fb3d252950:37987. 2024-11-18T07:54:51,177 DEBUG [RS:0;87fb3d252950:37987 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T07:54:51,177 DEBUG [RS:0;87fb3d252950:37987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:51,177 INFO [RS:0;87fb3d252950:37987 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T07:54:51,177 INFO [RS:0;87fb3d252950:37987 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T07:54:51,177 INFO [RS:0;87fb3d252950:37987 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T07:54:51,177 INFO [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T07:54:51,178 INFO [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-18T07:54:51,178 DEBUG [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-18T07:54:51,178 DEBUG [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T07:54:51,178 DEBUG [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T07:54:51,178 INFO [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T07:54:51,178 DEBUG [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T07:54:51,178 DEBUG [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T07:54:51,178 DEBUG [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T07:54:51,178 INFO [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-18T07:54:51,199 DEBUG [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/.tmp/info/e02d3c3e5e6146f79338e3573482a175 is 95, key is TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9./info:/1731916353242/DeleteFamily/seqid=0 2024-11-18T07:54:51,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742475_1651 (size=6514) 2024-11-18T07:54:51,259 INFO [regionserver/87fb3d252950:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T07:54:51,378 DEBUG [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T07:54:51,520 INFO [regionserver/87fb3d252950:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T07:54:51,520 INFO [regionserver/87fb3d252950:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T07:54:51,578 DEBUG [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T07:54:51,602 INFO [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/.tmp/info/e02d3c3e5e6146f79338e3573482a175 2024-11-18T07:54:51,623 DEBUG [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/.tmp/ns/0ecd7f53406146d2838ed590494fa13c is 93, key is TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9./ns:/1731916353192/DeleteFamily/seqid=0 2024-11-18T07:54:51,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742476_1652 (size=5906) 2024-11-18T07:54:51,779 DEBUG [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T07:54:51,979 DEBUG [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T07:54:52,026 INFO [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/.tmp/ns/0ecd7f53406146d2838ed590494fa13c 2024-11-18T07:54:52,049 DEBUG [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/.tmp/rep_barrier/215f1dffd6a44f0a8bd4e3bb30d468b1 is 102, key is TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9./rep_barrier:/1731916353192/DeleteFamily/seqid=0 2024-11-18T07:54:52,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742477_1653 (size=6025) 2024-11-18T07:54:52,052 INFO [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/.tmp/rep_barrier/215f1dffd6a44f0a8bd4e3bb30d468b1 2024-11-18T07:54:52,075 DEBUG [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/.tmp/table/dc12026cb287497baabbb5ad1c12bc1e is 96, key is TestAcidGuarantees,,1731916326719.1563b121b61f4db988d82be75c71ace9./table:/1731916353192/DeleteFamily/seqid=0 2024-11-18T07:54:52,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742478_1654 (size=5936) 2024-11-18T07:54:52,080 INFO [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/.tmp/table/dc12026cb287497baabbb5ad1c12bc1e 2024-11-18T07:54:52,085 DEBUG [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/.tmp/info/e02d3c3e5e6146f79338e3573482a175 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/info/e02d3c3e5e6146f79338e3573482a175 2024-11-18T07:54:52,089 INFO [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/info/e02d3c3e5e6146f79338e3573482a175, entries=12, sequenceid=90, filesize=6.4 K 2024-11-18T07:54:52,090 DEBUG [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/.tmp/ns/0ecd7f53406146d2838ed590494fa13c as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/ns/0ecd7f53406146d2838ed590494fa13c 2024-11-18T07:54:52,094 INFO [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/ns/0ecd7f53406146d2838ed590494fa13c, entries=8, sequenceid=90, filesize=5.8 K 2024-11-18T07:54:52,095 DEBUG [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/.tmp/rep_barrier/215f1dffd6a44f0a8bd4e3bb30d468b1 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/rep_barrier/215f1dffd6a44f0a8bd4e3bb30d468b1 2024-11-18T07:54:52,098 INFO [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/rep_barrier/215f1dffd6a44f0a8bd4e3bb30d468b1, entries=6, sequenceid=90, filesize=5.9 K 2024-11-18T07:54:52,099 DEBUG [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/.tmp/table/dc12026cb287497baabbb5ad1c12bc1e as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/table/dc12026cb287497baabbb5ad1c12bc1e 2024-11-18T07:54:52,103 INFO [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/table/dc12026cb287497baabbb5ad1c12bc1e, entries=7, sequenceid=90, filesize=5.8 K 2024-11-18T07:54:52,104 INFO [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 926ms, sequenceid=90, compaction requested=false 2024-11-18T07:54:52,114 DEBUG [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-18T07:54:52,114 DEBUG [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T07:54:52,115 INFO [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T07:54:52,115 DEBUG [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731916491178Running coprocessor pre-close hooks at 1731916491178Disabling compacts and flushes for region at 1731916491178Disabling writes for close at 1731916491178Obtaining lock to block concurrent updates at 1731916491178Preparing flush snapshotting stores in 1588230740 at 1731916491178Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731916491179 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731916491179Flushing 1588230740/info: creating writer at 1731916491180 (+1 ms)Flushing 1588230740/info: appending metadata at 1731916491198 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731916491198Flushing 1588230740/ns: creating writer at 1731916491606 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731916491622 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731916491622Flushing 1588230740/rep_barrier: creating writer at 1731916492030 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731916492049 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731916492049Flushing 1588230740/table: creating writer at 1731916492056 (+7 ms)Flushing 1588230740/table: appending metadata at 1731916492075 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731916492075Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@610771c1: reopening flushed file at 1731916492084 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1377cba4: reopening flushed file at 1731916492089 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19330193: reopening flushed file at 1731916492094 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18da42b0: reopening flushed file at 1731916492099 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 926ms, sequenceid=90, compaction requested=false at 1731916492104 (+5 ms)Writing region close event to WAL at 1731916492111 (+7 ms)Running coprocessor post-close hooks at 1731916492114 (+3 ms)Closed at 1731916492115 (+1 ms) 2024-11-18T07:54:52,115 DEBUG [RS_CLOSE_META-regionserver/87fb3d252950:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T07:54:52,179 INFO [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(976): stopping server 87fb3d252950,37987,1731916323128; all regions closed. 2024-11-18T07:54:52,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741834_1010 (size=25344) 2024-11-18T07:54:52,187 DEBUG [RS:0;87fb3d252950:37987 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/oldWALs 2024-11-18T07:54:52,187 INFO [RS:0;87fb3d252950:37987 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 87fb3d252950%2C37987%2C1731916323128.meta:.meta(num 1731916326141) 2024-11-18T07:54:52,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741833_1009 (size=19542624) 2024-11-18T07:54:52,191 DEBUG [RS:0;87fb3d252950:37987 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/oldWALs 2024-11-18T07:54:52,191 INFO [RS:0;87fb3d252950:37987 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 87fb3d252950%2C37987%2C1731916323128:(num 1731916325696) 2024-11-18T07:54:52,191 DEBUG [RS:0;87fb3d252950:37987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T07:54:52,191 INFO [RS:0;87fb3d252950:37987 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T07:54:52,191 INFO [RS:0;87fb3d252950:37987 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T07:54:52,191 INFO [RS:0;87fb3d252950:37987 {}] hbase.ChoreService(370): Chore service for: regionserver/87fb3d252950:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-18T07:54:52,192 INFO [RS:0;87fb3d252950:37987 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T07:54:52,192 INFO [regionserver/87fb3d252950:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T07:54:52,192 INFO [RS:0;87fb3d252950:37987 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37987 2024-11-18T07:54:52,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T07:54:52,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/87fb3d252950,37987,1731916323128 2024-11-18T07:54:52,244 INFO [RS:0;87fb3d252950:37987 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T07:54:52,245 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [87fb3d252950,37987,1731916323128] 2024-11-18T07:54:52,260 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/87fb3d252950,37987,1731916323128 already deleted, retry=false 2024-11-18T07:54:52,260 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 87fb3d252950,37987,1731916323128 expired; onlineServers=0 2024-11-18T07:54:52,261 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '87fb3d252950,42633,1731916322278' ***** 2024-11-18T07:54:52,261 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T07:54:52,261 INFO [M:0;87fb3d252950:42633 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T07:54:52,261 INFO [M:0;87fb3d252950:42633 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T07:54:52,261 DEBUG [M:0;87fb3d252950:42633 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T07:54:52,261 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T07:54:52,261 DEBUG [M:0;87fb3d252950:42633 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T07:54:52,261 INFO [M:0;87fb3d252950:42633 {}] hbase.ChoreService(370): Chore service for: master/87fb3d252950:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T07:54:52,261 DEBUG [master/87fb3d252950:0:becomeActiveMaster-HFileCleaner.small.0-1731916325371 {}] cleaner.HFileCleaner(306): Exit Thread[master/87fb3d252950:0:becomeActiveMaster-HFileCleaner.small.0-1731916325371,5,FailOnTimeoutGroup] 2024-11-18T07:54:52,261 INFO [M:0;87fb3d252950:42633 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T07:54:52,261 DEBUG [master/87fb3d252950:0:becomeActiveMaster-HFileCleaner.large.0-1731916325368 {}] cleaner.HFileCleaner(306): Exit Thread[master/87fb3d252950:0:becomeActiveMaster-HFileCleaner.large.0-1731916325368,5,FailOnTimeoutGroup] 2024-11-18T07:54:52,261 DEBUG [M:0;87fb3d252950:42633 {}] master.HMaster(1795): Stopping service threads 2024-11-18T07:54:52,261 INFO [M:0;87fb3d252950:42633 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T07:54:52,262 INFO [M:0;87fb3d252950:42633 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T07:54:52,262 ERROR [M:0;87fb3d252950:42633 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (741232665) connection to localhost/127.0.0.1:44371 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:44371,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-18T07:54:52,263 INFO [M:0;87fb3d252950:42633 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T07:54:52,263 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T07:54:52,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T07:54:52,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T07:54:52,269 DEBUG [M:0;87fb3d252950:42633 {}] zookeeper.ZKUtil(347): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T07:54:52,269 WARN [M:0;87fb3d252950:42633 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T07:54:52,269 INFO [M:0;87fb3d252950:42633 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/.lastflushedseqids 2024-11-18T07:54:52,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742479_1655 (size=72) 2024-11-18T07:54:52,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T07:54:52,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37987-0x1014e3765000001, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T07:54:52,353 INFO [RS:0;87fb3d252950:37987 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T07:54:52,353 INFO [RS:0;87fb3d252950:37987 {}] regionserver.HRegionServer(1031): Exiting; stopping=87fb3d252950,37987,1731916323128; zookeeper connection closed. 2024-11-18T07:54:52,353 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@38823ac3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@38823ac3 2024-11-18T07:54:52,354 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-18T07:54:52,682 INFO [M:0;87fb3d252950:42633 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T07:54:52,682 INFO [M:0;87fb3d252950:42633 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T07:54:52,683 DEBUG [M:0;87fb3d252950:42633 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T07:54:52,683 INFO [M:0;87fb3d252950:42633 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T07:54:52,683 DEBUG [M:0;87fb3d252950:42633 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T07:54:52,683 DEBUG [M:0;87fb3d252950:42633 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T07:54:52,683 DEBUG [M:0;87fb3d252950:42633 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T07:54:52,683 INFO [M:0;87fb3d252950:42633 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=763.99 KB heapSize=934.90 KB 2024-11-18T07:54:52,701 DEBUG [M:0;87fb3d252950:42633 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3314d0a304b74d6e8637316a9428f696 is 82, key is hbase:meta,,1/info:regioninfo/1731916326259/Put/seqid=0 2024-11-18T07:54:52,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742480_1656 (size=5672) 2024-11-18T07:54:52,725 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T07:54:52,725 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-18T07:54:52,725 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T07:54:53,108 INFO [M:0;87fb3d252950:42633 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2110 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3314d0a304b74d6e8637316a9428f696 2024-11-18T07:54:53,128 DEBUG [M:0;87fb3d252950:42633 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ca7c13c26f754fcebfc9712fc2af2b01 is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x8F/proc:d/1731916465572/Put/seqid=0 2024-11-18T07:54:53,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742481_1657 (size=44728) 2024-11-18T07:54:53,532 INFO [M:0;87fb3d252950:42633 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=763.43 KB at sequenceid=2110 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ca7c13c26f754fcebfc9712fc2af2b01 2024-11-18T07:54:53,535 INFO [M:0;87fb3d252950:42633 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ca7c13c26f754fcebfc9712fc2af2b01 2024-11-18T07:54:53,548 DEBUG [M:0;87fb3d252950:42633 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8fd44f4b18ed4abca4313ed9fc8ede77 is 69, key is 87fb3d252950,37987,1731916323128/rs:state/1731916325393/Put/seqid=0 2024-11-18T07:54:53,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073742482_1658 (size=5156) 2024-11-18T07:54:53,952 INFO [M:0;87fb3d252950:42633 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2110 (bloomFilter=true), to=hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8fd44f4b18ed4abca4313ed9fc8ede77 2024-11-18T07:54:53,955 DEBUG [M:0;87fb3d252950:42633 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3314d0a304b74d6e8637316a9428f696 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3314d0a304b74d6e8637316a9428f696 2024-11-18T07:54:53,958 INFO [M:0;87fb3d252950:42633 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3314d0a304b74d6e8637316a9428f696, entries=8, sequenceid=2110, filesize=5.5 K 2024-11-18T07:54:53,959 DEBUG [M:0;87fb3d252950:42633 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ca7c13c26f754fcebfc9712fc2af2b01 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ca7c13c26f754fcebfc9712fc2af2b01 2024-11-18T07:54:53,961 INFO [M:0;87fb3d252950:42633 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ca7c13c26f754fcebfc9712fc2af2b01 2024-11-18T07:54:53,961 INFO [M:0;87fb3d252950:42633 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ca7c13c26f754fcebfc9712fc2af2b01, entries=166, sequenceid=2110, filesize=43.7 K 2024-11-18T07:54:53,962 DEBUG [M:0;87fb3d252950:42633 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8fd44f4b18ed4abca4313ed9fc8ede77 as hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8fd44f4b18ed4abca4313ed9fc8ede77 2024-11-18T07:54:53,965 INFO [M:0;87fb3d252950:42633 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44371/user/jenkins/test-data/44119719-c24c-174d-12a6-09a8c5da95b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8fd44f4b18ed4abca4313ed9fc8ede77, entries=1, sequenceid=2110, filesize=5.0 K 2024-11-18T07:54:53,965 INFO [M:0;87fb3d252950:42633 {}] regionserver.HRegion(3140): Finished flush of dataSize ~763.99 KB/782326, heapSize ~934.60 KB/957032, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1282ms, sequenceid=2110, compaction requested=false 2024-11-18T07:54:53,972 INFO [M:0;87fb3d252950:42633 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T07:54:53,972 DEBUG [M:0;87fb3d252950:42633 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731916492682Disabling compacts and flushes for region at 1731916492682Disabling writes for close at 1731916492683 (+1 ms)Obtaining lock to block concurrent updates at 1731916492683Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731916492683Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=782326, getHeapSize=957272, getOffHeapSize=0, getCellsCount=2301 at 1731916492683Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731916492684 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731916492684Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731916492701 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731916492701Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731916493111 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731916493128 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731916493128Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731916493535 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731916493548 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731916493548Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@918c170: reopening flushed file at 1731916493954 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@294bc14b: reopening flushed file at 1731916493958 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9c3030: reopening flushed file at 1731916493961 (+3 ms)Finished flush of dataSize ~763.99 KB/782326, heapSize ~934.60 KB/957032, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1282ms, sequenceid=2110, compaction requested=false at 1731916493965 (+4 ms)Writing region close event to WAL at 1731916493972 (+7 ms)Closed at 1731916493972 2024-11-18T07:54:53,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40631 is added to blk_1073741830_1006 (size=920659) 2024-11-18T07:54:53,976 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T07:54:53,977 INFO [M:0;87fb3d252950:42633 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T07:54:53,977 INFO [M:0;87fb3d252950:42633 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42633 2024-11-18T07:54:53,977 INFO [M:0;87fb3d252950:42633 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T07:54:54,127 INFO [M:0;87fb3d252950:42633 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T07:54:54,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T07:54:54,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42633-0x1014e3765000000, quorum=127.0.0.1:61891, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T07:54:54,139 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@651ab874{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T07:54:54,141 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6bf72caa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T07:54:54,141 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T07:54:54,141 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@186a2d75{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T07:54:54,141 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72782cb0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/hadoop.log.dir/,STOPPED} 2024-11-18T07:54:54,149 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T07:54:54,149 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T07:54:54,149 WARN [BP-1093338534-172.17.0.2-1731916318289 heartbeating to localhost/127.0.0.1:44371 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T07:54:54,149 WARN [BP-1093338534-172.17.0.2-1731916318289 heartbeating to localhost/127.0.0.1:44371 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1093338534-172.17.0.2-1731916318289 (Datanode Uuid 392a6dfc-8655-4f40-a45a-57258804d6c2) service to localhost/127.0.0.1:44371 2024-11-18T07:54:54,154 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/cluster_477ae96c-b03e-42a8-3641-251bd626575f/data/data2/current/BP-1093338534-172.17.0.2-1731916318289 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T07:54:54,154 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/cluster_477ae96c-b03e-42a8-3641-251bd626575f/data/data1/current/BP-1093338534-172.17.0.2-1731916318289 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T07:54:54,155 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T07:54:54,170 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T07:54:54,171 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T07:54:54,171 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T07:54:54,171 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T07:54:54,172 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1779e2fc-12f0-2a16-99f4-9968501d1cf7/hadoop.log.dir/,STOPPED} 2024-11-18T07:54:54,194 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T07:54:54,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down